diff --git a/backup/src/boto/__init__.py b/backup/src/boto/__init__.py deleted file mode 100644 index d11b578..0000000 --- a/backup/src/boto/__init__.py +++ /dev/null @@ -1,542 +0,0 @@ -# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import boto -from boto.pyami.config import Config, BotoConfigLocations -from boto.storage_uri import BucketStorageUri, FileStorageUri -import boto.plugin -import os, re, sys -import logging -import logging.config -from boto.exception import InvalidUriError - -__version__ = '2.0b4' -Version = __version__ # for backware compatibility - -UserAgent = 'Boto/%s (%s)' % (__version__, sys.platform) -config = Config() - -def init_logging(): - for file in BotoConfigLocations: - try: - logging.config.fileConfig(os.path.expanduser(file)) - except: - pass - -class NullHandler(logging.Handler): - def emit(self, record): - pass - -log = logging.getLogger('boto') -log.addHandler(NullHandler()) -init_logging() - -# convenience function to set logging to a particular file -def set_file_logger(name, filepath, level=logging.INFO, format_string=None): - global log - if not format_string: - format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" - logger = logging.getLogger(name) - logger.setLevel(level) - fh = logging.FileHandler(filepath) - fh.setLevel(level) - formatter = logging.Formatter(format_string) - fh.setFormatter(formatter) - logger.addHandler(fh) - log = logger - -def set_stream_logger(name, level=logging.DEBUG, format_string=None): - global log - if not format_string: - format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" - logger = logging.getLogger(name) - logger.setLevel(level) - fh = logging.StreamHandler() - fh.setLevel(level) - formatter = logging.Formatter(format_string) - fh.setFormatter(formatter) - logger.addHandler(fh) - log = logger - -def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sqs.connection.SQSConnection` - :return: A connection to Amazon's SQS - """ - from boto.sqs.connection import SQSConnection - return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.s3.connection.S3Connection` - :return: A connection to Amazon's S3 - """ - from boto.s3.connection import S3Connection - return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_gs(gs_access_key_id=None, gs_secret_access_key=None, **kwargs): - """ - @type gs_access_key_id: string - @param gs_access_key_id: Your Google Storage Access Key ID - - @type gs_secret_access_key: string - @param gs_secret_access_key: Your Google Storage Secret Access Key - - @rtype: L{GSConnection} - @return: A connection to Google's Storage service - """ - from boto.gs.connection import GSConnection - return GSConnection(gs_access_key_id, gs_secret_access_key, **kwargs) - -def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.connection.EC2Connection` - :return: A connection to Amazon's EC2 - """ - from boto.ec2.connection import EC2Connection - return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_elb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.elb.ELBConnection` - :return: A connection to Amazon's Load Balancing Service - """ - from boto.ec2.elb import ELBConnection - return ELBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_autoscale(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` - :return: A connection to Amazon's Auto Scaling Service - """ - from boto.ec2.autoscale import AutoScaleConnection - return AutoScaleConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_cloudwatch(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` - :return: A connection to Amazon's EC2 Monitoring service - """ - from boto.ec2.cloudwatch import CloudWatchConnection - return CloudWatchConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sdb.connection.SDBConnection` - :return: A connection to Amazon's SDB - """ - from boto.sdb.connection import SDBConnection - return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.fps.connection.FPSConnection` - :return: A connection to FPS - """ - from boto.fps.connection import FPSConnection - return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_mturk(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.mturk.connection.MTurkConnection` - :return: A connection to MTurk - """ - from boto.mturk.connection import MTurkConnection - return MTurkConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.fps.connection.FPSConnection` - :return: A connection to FPS - """ - from boto.cloudfront import CloudFrontConnection - return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.vpc.VPCConnection` - :return: A connection to VPC - """ - from boto.vpc import VPCConnection - return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.rds.RDSConnection` - :return: A connection to RDS - """ - from boto.rds import RDSConnection - return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_emr(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.emr.EmrConnection` - :return: A connection to Elastic mapreduce - """ - from boto.emr import EmrConnection - return EmrConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_sns(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.sns.SNSConnection` - :return: A connection to Amazon's SNS - """ - from boto.sns import SNSConnection - return SNSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - - -def connect_iam(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.iam.IAMConnection` - :return: A connection to Amazon's IAM - """ - from boto.iam import IAMConnection - return IAMConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_route53(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.dns.Route53Connection` - :return: A connection to Amazon's Route53 DNS Service - """ - from boto.route53 import Route53Connection - return Route53Connection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_euca(host, aws_access_key_id=None, aws_secret_access_key=None, - port=8773, path='/services/Eucalyptus', is_secure=False, - **kwargs): - """ - Connect to a Eucalyptus service. - - :type host: string - :param host: the host name or ip address of the Eucalyptus server - - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ec2.connection.EC2Connection` - :return: A connection to Eucalyptus server - """ - from boto.ec2 import EC2Connection - from boto.ec2.regioninfo import RegionInfo - - reg = RegionInfo(name='eucalyptus', endpoint=host) - return EC2Connection(aws_access_key_id, aws_secret_access_key, - region=reg, port=port, path=path, - is_secure=is_secure, **kwargs) - -def connect_walrus(host, aws_access_key_id=None, aws_secret_access_key=None, - port=8773, path='/services/Walrus', is_secure=False, - **kwargs): - """ - Connect to a Walrus service. - - :type host: string - :param host: the host name or ip address of the Walrus server - - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.s3.connection.S3Connection` - :return: A connection to Walrus - """ - from boto.s3.connection import S3Connection - from boto.s3.connection import OrdinaryCallingFormat - - return S3Connection(aws_access_key_id, aws_secret_access_key, - host=host, port=port, path=path, - calling_format=OrdinaryCallingFormat(), - is_secure=is_secure, **kwargs) - -def connect_ses(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): - """ - :type aws_access_key_id: string - :param aws_access_key_id: Your AWS Access Key ID - - :type aws_secret_access_key: string - :param aws_secret_access_key: Your AWS Secret Access Key - - :rtype: :class:`boto.ses.SESConnection` - :return: A connection to Amazon's SES - """ - from boto.ses import SESConnection - return SESConnection(aws_access_key_id, aws_secret_access_key, **kwargs) - -def connect_ia(ia_access_key_id=None, ia_secret_access_key=None, - is_secure=False, **kwargs): - """ - Connect to the Internet Archive via their S3-like API. - - :type ia_access_key_id: string - :param ia_access_key_id: Your IA Access Key ID. This will also look in your - boto config file for an entry in the Credentials - section called "ia_access_key_id" - - :type ia_secret_access_key: string - :param ia_secret_access_key: Your IA Secret Access Key. This will also look in your - boto config file for an entry in the Credentials - section called "ia_secret_access_key" - - :rtype: :class:`boto.s3.connection.S3Connection` - :return: A connection to the Internet Archive - """ - from boto.s3.connection import S3Connection - from boto.s3.connection import OrdinaryCallingFormat - - access_key = config.get('Credentials', 'ia_access_key_id', - ia_access_key_id) - secret_key = config.get('Credentials', 'ia_secret_access_key', - ia_secret_access_key) - - return S3Connection(access_key, secret_key, - host='s3.us.archive.org', - calling_format=OrdinaryCallingFormat(), - is_secure=is_secure, **kwargs) - -def check_extensions(module_name, module_path): - """ - This function checks for extensions to boto modules. It should be called in the - __init__.py file of all boto modules. See: - http://code.google.com/p/boto/wiki/ExtendModules - - for details. - """ - option_name = '%s_extend' % module_name - version = config.get('Boto', option_name, None) - if version: - dirname = module_path[0] - path = os.path.join(dirname, version) - if os.path.isdir(path): - log.info('extending module %s with: %s' % (module_name, path)) - module_path.insert(0, path) - -_aws_cache = {} - -def _get_aws_conn(service): - global _aws_cache - conn = _aws_cache.get(service) - if not conn: - meth = getattr(sys.modules[__name__], 'connect_' + service) - conn = meth() - _aws_cache[service] = conn - return conn - -def lookup(service, name): - global _aws_cache - conn = _get_aws_conn(service) - obj = _aws_cache.get('.'.join((service, name)), None) - if not obj: - obj = conn.lookup(name) - _aws_cache['.'.join((service, name))] = obj - return obj - -def storage_uri(uri_str, default_scheme='file', debug=0, validate=True, - bucket_storage_uri_class=BucketStorageUri): - """ - Instantiate a StorageUri from a URI string. - - :type uri_str: string - :param uri_str: URI naming bucket + optional object. - :type default_scheme: string - :param default_scheme: default scheme for scheme-less URIs. - :type debug: int - :param debug: debug level to pass in to boto connection (range 0..2). - :type validate: bool - :param validate: whether to check for bucket name validity. - :type bucket_storage_uri_class: BucketStorageUri interface. - :param bucket_storage_uri_class: Allows mocking for unit tests. - - We allow validate to be disabled to allow caller - to implement bucket-level wildcarding (outside the boto library; - see gsutil). - - :rtype: :class:`boto.StorageUri` subclass - :return: StorageUri subclass for given URI. - - ``uri_str`` must be one of the following formats: - - * gs://bucket/name - * s3://bucket/name - * gs://bucket - * s3://bucket - * filename - - The last example uses the default scheme ('file', unless overridden) - """ - - # Manually parse URI components instead of using urlparse.urlparse because - # what we're calling URIs don't really fit the standard syntax for URIs - # (the latter includes an optional host/net location part). - end_scheme_idx = uri_str.find('://') - if end_scheme_idx == -1: - # Check for common error: user specifies gs:bucket instead - # of gs://bucket. Some URI parsers allow this, but it can cause - # confusion for callers, so we don't. - if uri_str.find(':') != -1: - raise InvalidUriError('"%s" contains ":" instead of "://"' % uri_str) - scheme = default_scheme.lower() - path = uri_str - else: - scheme = uri_str[0:end_scheme_idx].lower() - path = uri_str[end_scheme_idx + 3:] - - if scheme not in ['file', 's3', 'gs']: - raise InvalidUriError('Unrecognized scheme "%s"' % scheme) - if scheme == 'file': - # For file URIs we have no bucket name, and use the complete path - # (minus 'file://') as the object name. - return FileStorageUri(path, debug) - else: - path_parts = path.split('/', 1) - bucket_name = path_parts[0] - if (validate and bucket_name and - # Disallow buckets violating charset or not [3..255] chars total. - (not re.match('^[a-z0-9][a-z0-9\._-]{1,253}[a-z0-9]$', bucket_name) - # Disallow buckets with individual DNS labels longer than 63. - or re.search('[-_a-z0-9]{64}', bucket_name))): - raise InvalidUriError('Invalid bucket name in URI "%s"' % uri_str) - # If enabled, ensure the bucket name is valid, to avoid possibly - # confusing other parts of the code. (For example if we didn't - # catch bucket names containing ':', when a user tried to connect to - # the server with that name they might get a confusing error about - # non-integer port numbers.) - object_name = '' - if len(path_parts) > 1: - object_name = path_parts[1] - return bucket_storage_uri_class(scheme, bucket_name, object_name, debug) - -def storage_uri_for_key(key): - """Returns a StorageUri for the given key. - - :type key: :class:`boto.s3.key.Key` or subclass - :param key: URI naming bucket + optional object. - """ - if not isinstance(key, boto.s3.key.Key): - raise InvalidUriError('Requested key (%s) is not a subclass of ' - 'boto.s3.key.Key' % str(type(key))) - prov_name = key.bucket.connection.provider.get_provider_name() - uri_str = '%s://%s/%s' % (prov_name, key.bucket.name, key.name) - return storage_uri(uri_str) - -boto.plugin.load_plugins(config) diff --git a/backup/src/boto/auth.py b/backup/src/boto/auth.py deleted file mode 100644 index 6c6c1f2..0000000 --- a/backup/src/boto/auth.py +++ /dev/null @@ -1,319 +0,0 @@ -# Copyright 2010 Google Inc. -# Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2011, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -""" -Handles authentication required to AWS and GS -""" - -import base64 -import boto -import boto.auth_handler -import boto.exception -import boto.plugin -import boto.utils -import hmac -import sys -import time -import urllib - -from boto.auth_handler import AuthHandler -from boto.exception import BotoClientError -# -# the following is necessary because of the incompatibilities -# between Python 2.4, 2.5, and 2.6 as well as the fact that some -# people running 2.4 have installed hashlib as a separate module -# this fix was provided by boto user mccormix. -# see: http://code.google.com/p/boto/issues/detail?id=172 -# for more details. -# -try: - from hashlib import sha1 as sha - from hashlib import sha256 as sha256 - - if sys.version[:3] == "2.4": - # we are using an hmac that expects a .new() method. - class Faker: - def __init__(self, which): - self.which = which - self.digest_size = self.which().digest_size - - def new(self, *args, **kwargs): - return self.which(*args, **kwargs) - - sha = Faker(sha) - sha256 = Faker(sha256) - -except ImportError: - import sha - sha256 = None - -class HmacKeys(object): - """Key based Auth handler helper.""" - - def __init__(self, host, config, provider): - if provider.access_key is None or provider.secret_key is None: - raise boto.auth_handler.NotReadyToAuthenticate() - self._provider = provider - self._hmac = hmac.new(self._provider.secret_key, digestmod=sha) - if sha256: - self._hmac_256 = hmac.new(self._provider.secret_key, digestmod=sha256) - else: - self._hmac_256 = None - - def algorithm(self): - if self._hmac_256: - return 'HmacSHA256' - else: - return 'HmacSHA1' - - def sign_string(self, string_to_sign): - boto.log.debug('Canonical: %s' % string_to_sign) - if self._hmac_256: - hmac = self._hmac_256.copy() - else: - hmac = self._hmac.copy() - hmac.update(string_to_sign) - return base64.encodestring(hmac.digest()).strip() - -class HmacAuthV1Handler(AuthHandler, HmacKeys): - """ Implements the HMAC request signing used by S3 and GS.""" - - capability = ['hmac-v1', 's3'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - self._hmac_256 = None - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - method = http_request.method - auth_path = http_request.auth_path - if not headers.has_key('Date'): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime()) - - c_string = boto.utils.canonical_string(method, auth_path, headers, - None, self._provider) - b64_hmac = self.sign_string(c_string) - auth_hdr = self._provider.auth_header - headers['Authorization'] = ("%s %s:%s" % - (auth_hdr, - self._provider.access_key, b64_hmac)) - -class HmacAuthV2Handler(AuthHandler, HmacKeys): - """ - Implements the simplified HMAC authorization used by CloudFront. - """ - capability = ['hmac-v2', 'cloudfront'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - self._hmac_256 = None - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - if not headers.has_key('Date'): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime()) - - b64_hmac = self.sign_string(headers['Date']) - auth_hdr = self._provider.auth_header - headers['Authorization'] = ("%s %s:%s" % - (auth_hdr, - self._provider.access_key, b64_hmac)) - -class HmacAuthV3Handler(AuthHandler, HmacKeys): - """Implements the new Version 3 HMAC authorization used by Route53.""" - - capability = ['hmac-v3', 'route53', 'ses'] - - def __init__(self, host, config, provider): - AuthHandler.__init__(self, host, config, provider) - HmacKeys.__init__(self, host, config, provider) - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - if not headers.has_key('Date'): - headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", - time.gmtime()) - - b64_hmac = self.sign_string(headers['Date']) - s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key - s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac) - headers['X-Amzn-Authorization'] = s - -class QuerySignatureHelper(HmacKeys): - """Helper for Query signature based Auth handler. - - Concrete sub class need to implement _calc_sigature method. - """ - - def add_auth(self, http_request, **kwargs): - headers = http_request.headers - params = http_request.params - params['AWSAccessKeyId'] = self._provider.access_key - params['SignatureVersion'] = self.SignatureVersion - params['Timestamp'] = boto.utils.get_ts() - qs, signature = self._calc_signature( - http_request.params, http_request.method, - http_request.path, http_request.host) - boto.log.debug('query_string: %s Signature: %s' % (qs, signature)) - if http_request.method == 'POST': - headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' - http_request.body = qs + '&Signature=' + urllib.quote(signature) - else: - http_request.body = '' - http_request.path = (http_request.path + '?' + qs + '&Signature=' + urllib.quote(signature)) - # Now that query params are part of the path, clear the 'params' field - # in request. - http_request.params = {} - -class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler): - """Class SQS query signature based Auth handler.""" - - SignatureVersion = 0 - capability = ['sign-v0'] - - def _calc_signature(self, params, *args): - boto.log.debug('using _calc_signature_0') - hmac = self._hmac.copy() - s = params['Action'] + params['Timestamp'] - hmac.update(s) - keys = params.keys() - keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - val = bot.utils.get_utf8_value(params[key]) - pairs.append(key + '=' + urllib.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - -class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler): - """ - Provides Query Signature V1 Authentication. - """ - - SignatureVersion = 1 - capability = ['sign-v1', 'mturk'] - - def _calc_signature(self, params, *args): - boto.log.debug('using _calc_signature_1') - hmac = self._hmac.copy() - keys = params.keys() - keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) - pairs = [] - for key in keys: - hmac.update(key) - val = boto.utils.get_utf8_value(params[key]) - hmac.update(val) - pairs.append(key + '=' + urllib.quote(val)) - qs = '&'.join(pairs) - return (qs, base64.b64encode(hmac.digest())) - -class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler): - """Provides Query Signature V2 Authentication.""" - - SignatureVersion = 2 - capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs', - 'sdb', 'iam', 'rds', 'sns', 'sqs'] - - def _calc_signature(self, params, verb, path, server_name): - boto.log.debug('using _calc_signature_2') - string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path) - if self._hmac_256: - hmac = self._hmac_256.copy() - params['SignatureMethod'] = 'HmacSHA256' - else: - hmac = self._hmac.copy() - params['SignatureMethod'] = 'HmacSHA1' - keys = params.keys() - keys.sort() - pairs = [] - for key in keys: - val = boto.utils.get_utf8_value(params[key]) - pairs.append(urllib.quote(key, safe='') + '=' + - urllib.quote(val, safe='-_~')) - qs = '&'.join(pairs) - boto.log.debug('query string: %s' % qs) - string_to_sign += qs - boto.log.debug('string_to_sign: %s' % string_to_sign) - hmac.update(string_to_sign) - b64 = base64.b64encode(hmac.digest()) - boto.log.debug('len(b64)=%d' % len(b64)) - boto.log.debug('base64 encoded digest: %s' % b64) - return (qs, b64) - - -def get_auth_handler(host, config, provider, requested_capability=None): - """Finds an AuthHandler that is ready to authenticate. - - Lists through all the registered AuthHandlers to find one that is willing - to handle for the requested capabilities, config and provider. - - :type host: string - :param host: The name of the host - - :type config: - :param config: - - :type provider: - :param provider: - - Returns: - An implementation of AuthHandler. - - Raises: - boto.exception.NoAuthHandlerFound: - boto.exception.TooManyAuthHandlerReadyToAuthenticate: - """ - ready_handlers = [] - auth_handlers = boto.plugin.get_plugin(AuthHandler, requested_capability) - total_handlers = len(auth_handlers) - for handler in auth_handlers: - try: - ready_handlers.append(handler(host, config, provider)) - except boto.auth_handler.NotReadyToAuthenticate: - pass - - if not ready_handlers: - checked_handlers = auth_handlers - names = [handler.__name__ for handler in checked_handlers] - raise boto.exception.NoAuthHandlerFound( - 'No handler was ready to authenticate. %d handlers were checked.' - ' %s ' % (len(names), str(names))) - - if len(ready_handlers) > 1: - # NOTE: Even though it would be nice to accept more than one handler - # by using one of the many ready handlers, we are never sure that each - # of them are referring to the same storage account. Since we cannot - # easily guarantee that, it is always safe to fail, rather than operate - # on the wrong account. - names = [handler.__class__.__name__ for handler in ready_handlers] - raise boto.exception.TooManyAuthHandlerReadyToAuthenticate( - '%d AuthHandlers ready to authenticate, ' - 'only 1 expected: %s' % (len(names), str(names))) - - return ready_handlers[0] diff --git a/backup/src/boto/auth_handler.py b/backup/src/boto/auth_handler.py deleted file mode 100644 index ab2d317..0000000 --- a/backup/src/boto/auth_handler.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Defines an interface which all Auth handlers need to implement. -""" - -from plugin import Plugin - -class NotReadyToAuthenticate(Exception): - pass - -class AuthHandler(Plugin): - - capability = [] - - def __init__(self, host, config, provider): - """Constructs the handlers. - :type host: string - :param host: The host to which the request is being sent. - - :type config: boto.pyami.Config - :param config: Boto configuration. - - :type provider: boto.provider.Provider - :param provider: Provider details. - - Raises: - NotReadyToAuthenticate: if this handler is not willing to - authenticate for the given provider and config. - """ - pass - - def add_auth(self, http_request): - """Invoked to add authentication details to request. - - :type http_request: boto.connection.HTTPRequest - :param http_request: HTTP request that needs to be authenticated. - """ - pass diff --git a/backup/src/boto/cloudfront/__init__.py b/backup/src/boto/cloudfront/__init__.py deleted file mode 100644 index bd02b00..0000000 --- a/backup/src/boto/cloudfront/__init__.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import xml.sax -import time -import boto -from boto.connection import AWSAuthConnection -from boto import handler -from boto.cloudfront.distribution import Distribution, DistributionSummary, DistributionConfig -from boto.cloudfront.distribution import StreamingDistribution, StreamingDistributionSummary, StreamingDistributionConfig -from boto.cloudfront.identity import OriginAccessIdentity -from boto.cloudfront.identity import OriginAccessIdentitySummary -from boto.cloudfront.identity import OriginAccessIdentityConfig -from boto.cloudfront.invalidation import InvalidationBatch -from boto.resultset import ResultSet -from boto.cloudfront.exception import CloudFrontServerError - -class CloudFrontConnection(AWSAuthConnection): - - DefaultHost = 'cloudfront.amazonaws.com' - Version = '2010-11-01' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - port=None, proxy=None, proxy_port=None, - host=DefaultHost, debug=0): - AWSAuthConnection.__init__(self, host, - aws_access_key_id, aws_secret_access_key, - True, port, proxy, proxy_port, debug=debug) - - def get_etag(self, response): - response_headers = response.msg - for key in response_headers.keys(): - if key.lower() == 'etag': - return response_headers[key] - return None - - def _required_auth_capability(self): - return ['cloudfront'] - - # Generics - - def _get_all_objects(self, resource, tags): - if not tags: - tags=[('DistributionSummary', DistributionSummary)] - response = self.make_request('GET', '/%s/%s' % (self.Version, resource)) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise CloudFrontServerError(response.status, response.reason, body) - rs = ResultSet(tags) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - - def _get_info(self, id, resource, dist_class): - uri = '/%s/%s/%s' % (self.Version, resource, id) - response = self.make_request('GET', uri) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise CloudFrontServerError(response.status, response.reason, body) - d = dist_class(connection=self) - response_headers = response.msg - for key in response_headers.keys(): - if key.lower() == 'etag': - d.etag = response_headers[key] - h = handler.XmlHandler(d, self) - xml.sax.parseString(body, h) - return d - - def _get_config(self, id, resource, config_class): - uri = '/%s/%s/%s/config' % (self.Version, resource, id) - response = self.make_request('GET', uri) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise CloudFrontServerError(response.status, response.reason, body) - d = config_class(connection=self) - d.etag = self.get_etag(response) - h = handler.XmlHandler(d, self) - xml.sax.parseString(body, h) - return d - - def _set_config(self, distribution_id, etag, config): - if isinstance(config, StreamingDistributionConfig): - resource = 'streaming-distribution' - else: - resource = 'distribution' - uri = '/%s/%s/%s/config' % (self.Version, resource, distribution_id) - headers = {'If-Match' : etag, 'Content-Type' : 'text/xml'} - response = self.make_request('PUT', uri, headers, config.to_xml()) - body = response.read() - boto.log.debug(body) - if response.status != 200: - raise CloudFrontServerError(response.status, response.reason, body) - return self.get_etag(response) - - def _create_object(self, config, resource, dist_class): - response = self.make_request('POST', '/%s/%s' % (self.Version, resource), - {'Content-Type' : 'text/xml'}, data=config.to_xml()) - body = response.read() - boto.log.debug(body) - if response.status == 201: - d = dist_class(connection=self) - h = handler.XmlHandler(d, self) - xml.sax.parseString(body, h) - d.etag = self.get_etag(response) - return d - else: - raise CloudFrontServerError(response.status, response.reason, body) - - def _delete_object(self, id, etag, resource): - uri = '/%s/%s/%s' % (self.Version, resource, id) - response = self.make_request('DELETE', uri, {'If-Match' : etag}) - body = response.read() - boto.log.debug(body) - if response.status != 204: - raise CloudFrontServerError(response.status, response.reason, body) - - # Distributions - - def get_all_distributions(self): - tags=[('DistributionSummary', DistributionSummary)] - return self._get_all_objects('distribution', tags) - - def get_distribution_info(self, distribution_id): - return self._get_info(distribution_id, 'distribution', Distribution) - - def get_distribution_config(self, distribution_id): - return self._get_config(distribution_id, 'distribution', - DistributionConfig) - - def set_distribution_config(self, distribution_id, etag, config): - return self._set_config(distribution_id, etag, config) - - def create_distribution(self, origin, enabled, caller_reference='', - cnames=None, comment=''): - config = DistributionConfig(origin=origin, enabled=enabled, - caller_reference=caller_reference, - cnames=cnames, comment=comment) - return self._create_object(config, 'distribution', Distribution) - - def delete_distribution(self, distribution_id, etag): - return self._delete_object(distribution_id, etag, 'distribution') - - # Streaming Distributions - - def get_all_streaming_distributions(self): - tags=[('StreamingDistributionSummary', StreamingDistributionSummary)] - return self._get_all_objects('streaming-distribution', tags) - - def get_streaming_distribution_info(self, distribution_id): - return self._get_info(distribution_id, 'streaming-distribution', - StreamingDistribution) - - def get_streaming_distribution_config(self, distribution_id): - return self._get_config(distribution_id, 'streaming-distribution', - StreamingDistributionConfig) - - def set_streaming_distribution_config(self, distribution_id, etag, config): - return self._set_config(distribution_id, etag, config) - - def create_streaming_distribution(self, origin, enabled, - caller_reference='', - cnames=None, comment=''): - config = StreamingDistributionConfig(origin=origin, enabled=enabled, - caller_reference=caller_reference, - cnames=cnames, comment=comment) - return self._create_object(config, 'streaming-distribution', - StreamingDistribution) - - def delete_streaming_distribution(self, distribution_id, etag): - return self._delete_object(distribution_id, etag, 'streaming-distribution') - - # Origin Access Identity - - def get_all_origin_access_identity(self): - tags=[('CloudFrontOriginAccessIdentitySummary', - OriginAccessIdentitySummary)] - return self._get_all_objects('origin-access-identity/cloudfront', tags) - - def get_origin_access_identity_info(self, access_id): - return self._get_info(access_id, 'origin-access-identity/cloudfront', - OriginAccessIdentity) - - def get_origin_access_identity_config(self, access_id): - return self._get_config(access_id, - 'origin-access-identity/cloudfront', - OriginAccessIdentityConfig) - - def set_origin_access_identity_config(self, access_id, - etag, config): - return self._set_config(access_id, etag, config) - - def create_origin_access_identity(self, caller_reference='', comment=''): - config = OriginAccessIdentityConfig(caller_reference=caller_reference, - comment=comment) - return self._create_object(config, 'origin-access-identity/cloudfront', - OriginAccessIdentity) - - def delete_origin_access_identity(self, access_id, etag): - return self._delete_object(access_id, etag, - 'origin-access-identity/cloudfront') - - # Object Invalidation - - def create_invalidation_request(self, distribution_id, paths, - caller_reference=None): - """Creates a new invalidation request - :see: http://goo.gl/8vECq - """ - # We allow you to pass in either an array or - # an InvalidationBatch object - if not isinstance(paths, InvalidationBatch): - paths = InvalidationBatch(paths) - paths.connection = self - uri = '/%s/distribution/%s/invalidation' % (self.Version, - distribution_id) - response = self.make_request('POST', uri, - {'Content-Type' : 'text/xml'}, - data=paths.to_xml()) - body = response.read() - if response.status == 201: - h = handler.XmlHandler(paths, self) - xml.sax.parseString(body, h) - return paths - else: - raise CloudFrontServerError(response.status, response.reason, body) - diff --git a/backup/src/boto/cloudfront/distribution.py b/backup/src/boto/cloudfront/distribution.py deleted file mode 100644 index ed245cb..0000000 --- a/backup/src/boto/cloudfront/distribution.py +++ /dev/null @@ -1,540 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import uuid -from boto.cloudfront.identity import OriginAccessIdentity -from boto.cloudfront.object import Object, StreamingObject -from boto.cloudfront.signers import ActiveTrustedSigners, TrustedSigners -from boto.cloudfront.logging import LoggingInfo -from boto.cloudfront.origin import S3Origin, CustomOrigin -from boto.s3.acl import ACL - -class DistributionConfig: - - def __init__(self, connection=None, origin=None, enabled=False, - caller_reference='', cnames=None, comment='', - trusted_signers=None, default_root_object=None, - logging=None): - """ - :param origin: Origin information to associate with the - distribution. If your distribution will use - an Amazon S3 origin, then this should be an - S3Origin object. If your distribution will use - a custom origin (non Amazon S3), then this - should be a CustomOrigin object. - :type origin: :class:`boto.cloudfront.origin.S3Origin` or - :class:`boto.cloudfront.origin.CustomOrigin` - - :param enabled: Whether the distribution is enabled to accept - end user requests for content. - :type enabled: bool - - :param caller_reference: A unique number that ensures the - request can't be replayed. If no - caller_reference is provided, boto - will generate a type 4 UUID for use - as the caller reference. - :type enabled: str - - :param cnames: A CNAME alias you want to associate with this - distribution. You can have up to 10 CNAME aliases - per distribution. - :type enabled: array of str - - :param comment: Any comments you want to include about the - distribution. - :type comment: str - - :param trusted_signers: Specifies any AWS accounts you want to - permit to create signed URLs for private - content. If you want the distribution to - use signed URLs, this should contain a - TrustedSigners object; if you want the - distribution to use basic URLs, leave - this None. - :type trusted_signers: :class`boto.cloudfront.signers.TrustedSigners` - - :param default_root_object: Designates a default root object. - Only include a DefaultRootObject value - if you are going to assign a default - root object for the distribution. - :type comment: str - - :param logging: Controls whether access logs are written for the - distribution. If you want to turn on access logs, - this should contain a LoggingInfo object; otherwise - it should contain None. - :type logging: :class`boto.cloudfront.logging.LoggingInfo` - - """ - self.connection = connection - self.origin = origin - self.enabled = enabled - if caller_reference: - self.caller_reference = caller_reference - else: - self.caller_reference = str(uuid.uuid4()) - self.cnames = [] - if cnames: - self.cnames = cnames - self.comment = comment - self.trusted_signers = trusted_signers - self.logging = None - self.default_root_object = default_root_object - - def to_xml(self): - s = '\n' - s += '\n' - if self.origin: - s += self.origin.to_xml() - s += ' %s\n' % self.caller_reference - for cname in self.cnames: - s += ' %s\n' % cname - if self.comment: - s += ' %s\n' % self.comment - s += ' ' - if self.enabled: - s += 'true' - else: - s += 'false' - s += '\n' - if self.trusted_signers: - s += '\n' - for signer in self.trusted_signers: - if signer == 'Self': - s += ' \n' - else: - s += ' %s\n' % signer - s += '\n' - if self.logging: - s += '\n' - s += ' %s\n' % self.logging.bucket - s += ' %s\n' % self.logging.prefix - s += '\n' - if self.default_root_object: - dro = self.default_root_object - s += '%s\n' % dro - s += '\n' - return s - - def startElement(self, name, attrs, connection): - if name == 'TrustedSigners': - self.trusted_signers = TrustedSigners() - return self.trusted_signers - elif name == 'Logging': - self.logging = LoggingInfo() - return self.logging - elif name == 'S3Origin': - self.origin = S3Origin() - return self.origin - elif name == 'CustomOrigin': - self.origin = CustomOrigin() - return self.origin - else: - return None - - def endElement(self, name, value, connection): - if name == 'CNAME': - self.cnames.append(value) - elif name == 'Comment': - self.comment = value - elif name == 'Enabled': - if value.lower() == 'true': - self.enabled = True - else: - self.enabled = False - elif name == 'CallerReference': - self.caller_reference = value - elif name == 'DefaultRootObject': - self.default_root_object = value - else: - setattr(self, name, value) - -class StreamingDistributionConfig(DistributionConfig): - - def __init__(self, connection=None, origin='', enabled=False, - caller_reference='', cnames=None, comment='', - trusted_signers=None, logging=None): - DistributionConfig.__init__(self, connection=connection, - origin=origin, enabled=enabled, - caller_reference=caller_reference, - cnames=cnames, comment=comment, - trusted_signers=trusted_signers, - logging=logging) - def to_xml(self): - s = '\n' - s += '\n' - if self.origin: - s += self.origin.to_xml() - s += ' %s\n' % self.caller_reference - for cname in self.cnames: - s += ' %s\n' % cname - if self.comment: - s += ' %s\n' % self.comment - s += ' ' - if self.enabled: - s += 'true' - else: - s += 'false' - s += '\n' - if self.trusted_signers: - s += '\n' - for signer in self.trusted_signers: - if signer == 'Self': - s += ' \n' - else: - s += ' %s\n' % signer - s += '\n' - if self.logging: - s += '\n' - s += ' %s\n' % self.logging.bucket - s += ' %s\n' % self.logging.prefix - s += '\n' - s += '\n' - return s - -class DistributionSummary: - - def __init__(self, connection=None, domain_name='', id='', - last_modified_time=None, status='', origin=None, - cname='', comment='', enabled=False): - self.connection = connection - self.domain_name = domain_name - self.id = id - self.last_modified_time = last_modified_time - self.status = status - self.origin = origin - self.enabled = enabled - self.cnames = [] - if cname: - self.cnames.append(cname) - self.comment = comment - self.trusted_signers = None - self.etag = None - self.streaming = False - - def startElement(self, name, attrs, connection): - if name == 'TrustedSigners': - self.trusted_signers = TrustedSigners() - return self.trusted_signers - elif name == 'S3Origin': - self.origin = S3Origin() - return self.origin - elif name == 'CustomOrigin': - self.origin = CustomOrigin() - return self.origin - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'Status': - self.status = value - elif name == 'LastModifiedTime': - self.last_modified_time = value - elif name == 'DomainName': - self.domain_name = value - elif name == 'Origin': - self.origin = value - elif name == 'CNAME': - self.cnames.append(value) - elif name == 'Comment': - self.comment = value - elif name == 'Enabled': - if value.lower() == 'true': - self.enabled = True - else: - self.enabled = False - elif name == 'StreamingDistributionSummary': - self.streaming = True - else: - setattr(self, name, value) - - def get_distribution(self): - return self.connection.get_distribution_info(self.id) - -class StreamingDistributionSummary(DistributionSummary): - - def get_distribution(self): - return self.connection.get_streaming_distribution_info(self.id) - -class Distribution: - - def __init__(self, connection=None, config=None, domain_name='', - id='', last_modified_time=None, status=''): - self.connection = connection - self.config = config - self.domain_name = domain_name - self.id = id - self.last_modified_time = last_modified_time - self.status = status - self.active_signers = None - self.etag = None - self._bucket = None - self._object_class = Object - - def startElement(self, name, attrs, connection): - if name == 'DistributionConfig': - self.config = DistributionConfig() - return self.config - elif name == 'ActiveTrustedSigners': - self.active_signers = ActiveTrustedSigners() - return self.active_signers - else: - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'LastModifiedTime': - self.last_modified_time = value - elif name == 'Status': - self.status = value - elif name == 'DomainName': - self.domain_name = value - else: - setattr(self, name, value) - - def update(self, enabled=None, cnames=None, comment=None): - """ - Update the configuration of the Distribution. The only values - of the DistributionConfig that can be updated are: - - * CNAMES - * Comment - * Whether the Distribution is enabled or not - - :type enabled: bool - :param enabled: Whether the Distribution is active or not. - - :type cnames: list of str - :param cnames: The DNS CNAME's associated with this - Distribution. Maximum of 10 values. - - :type comment: str or unicode - :param comment: The comment associated with the Distribution. - - """ - new_config = DistributionConfig(self.connection, self.config.origin, - self.config.enabled, self.config.caller_reference, - self.config.cnames, self.config.comment, - self.config.trusted_signers, - self.config.default_root_object) - if enabled != None: - new_config.enabled = enabled - if cnames != None: - new_config.cnames = cnames - if comment != None: - new_config.comment = comment - self.etag = self.connection.set_distribution_config(self.id, self.etag, new_config) - self.config = new_config - self._object_class = Object - - def enable(self): - """ - Deactivate the Distribution. A convenience wrapper around - the update method. - """ - self.update(enabled=True) - - def disable(self): - """ - Activate the Distribution. A convenience wrapper around - the update method. - """ - self.update(enabled=False) - - def delete(self): - """ - Delete this CloudFront Distribution. The content - associated with the Distribution is not deleted from - the underlying Origin bucket in S3. - """ - self.connection.delete_distribution(self.id, self.etag) - - def _get_bucket(self): - if not self._bucket: - bucket_name = self.config.origin.replace('.s3.amazonaws.com', '') - from boto.s3.connection import S3Connection - s3 = S3Connection(self.connection.aws_access_key_id, - self.connection.aws_secret_access_key, - proxy=self.connection.proxy, - proxy_port=self.connection.proxy_port, - proxy_user=self.connection.proxy_user, - proxy_pass=self.connection.proxy_pass) - self._bucket = s3.get_bucket(bucket_name) - self._bucket.distribution = self - self._bucket.set_key_class(self._object_class) - return self._bucket - - def get_objects(self): - """ - Return a list of all content objects in this distribution. - - :rtype: list of :class:`boto.cloudfront.object.Object` - :return: The content objects - """ - bucket = self._get_bucket() - objs = [] - for key in bucket: - objs.append(key) - return objs - - def set_permissions(self, object, replace=False): - """ - Sets the S3 ACL grants for the given object to the appropriate - value based on the type of Distribution. If the Distribution - is serving private content the ACL will be set to include the - Origin Access Identity associated with the Distribution. If - the Distribution is serving public content the content will - be set up with "public-read". - - :type object: :class:`boto.cloudfront.object.Object` - :param enabled: The Object whose ACL is being set - - :type replace: bool - :param replace: If False, the Origin Access Identity will be - appended to the existing ACL for the object. - If True, the ACL for the object will be - completely replaced with one that grants - READ permission to the Origin Access Identity. - - """ - if isinstance(self.config.origin, S3Origin): - if self.config.origin.origin_access_identity: - id = self.config.origin.origin_access_identity.split('/')[-1] - oai = self.connection.get_origin_access_identity_info(id) - policy = object.get_acl() - if replace: - policy.acl = ACL() - policy.acl.add_user_grant('READ', oai.s3_user_id) - object.set_acl(policy) - else: - object.set_canned_acl('public-read') - - def set_permissions_all(self, replace=False): - """ - Sets the S3 ACL grants for all objects in the Distribution - to the appropriate value based on the type of Distribution. - - :type replace: bool - :param replace: If False, the Origin Access Identity will be - appended to the existing ACL for the object. - If True, the ACL for the object will be - completely replaced with one that grants - READ permission to the Origin Access Identity. - - """ - bucket = self._get_bucket() - for key in bucket: - self.set_permissions(key, replace) - - def add_object(self, name, content, headers=None, replace=True): - """ - Adds a new content object to the Distribution. The content - for the object will be copied to a new Key in the S3 Bucket - and the permissions will be set appropriately for the type - of Distribution. - - :type name: str or unicode - :param name: The name or key of the new object. - - :type content: file-like object - :param content: A file-like object that contains the content - for the new object. - - :type headers: dict - :param headers: A dictionary containing additional headers - you would like associated with the new - object in S3. - - :rtype: :class:`boto.cloudfront.object.Object` - :return: The newly created object. - """ - if self.config.origin_access_identity: - policy = 'private' - else: - policy = 'public-read' - bucket = self._get_bucket() - object = bucket.new_key(name) - object.set_contents_from_file(content, headers=headers, policy=policy) - if self.config.origin_access_identity: - self.set_permissions(object, replace) - return object - -class StreamingDistribution(Distribution): - - def __init__(self, connection=None, config=None, domain_name='', - id='', last_modified_time=None, status=''): - Distribution.__init__(self, connection, config, domain_name, - id, last_modified_time, status) - self._object_class = StreamingObject - - def startElement(self, name, attrs, connection): - if name == 'StreamingDistributionConfig': - self.config = StreamingDistributionConfig() - return self.config - else: - return Distribution.startElement(self, name, attrs, connection) - - def update(self, enabled=None, cnames=None, comment=None): - """ - Update the configuration of the StreamingDistribution. The only values - of the StreamingDistributionConfig that can be updated are: - - * CNAMES - * Comment - * Whether the Distribution is enabled or not - - :type enabled: bool - :param enabled: Whether the StreamingDistribution is active or not. - - :type cnames: list of str - :param cnames: The DNS CNAME's associated with this - Distribution. Maximum of 10 values. - - :type comment: str or unicode - :param comment: The comment associated with the Distribution. - - """ - new_config = StreamingDistributionConfig(self.connection, - self.config.origin, - self.config.enabled, - self.config.caller_reference, - self.config.cnames, - self.config.comment, - self.config.trusted_signers) - if enabled != None: - new_config.enabled = enabled - if cnames != None: - new_config.cnames = cnames - if comment != None: - new_config.comment = comment - self.etag = self.connection.set_streaming_distribution_config(self.id, - self.etag, - new_config) - self.config = new_config - self._object_class = StreamingObject - - def delete(self): - self.connection.delete_streaming_distribution(self.id, self.etag) - - diff --git a/backup/src/boto/cloudfront/exception.py b/backup/src/boto/cloudfront/exception.py deleted file mode 100644 index 7680642..0000000 --- a/backup/src/boto/cloudfront/exception.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.exception import BotoServerError - -class CloudFrontServerError(BotoServerError): - - pass diff --git a/backup/src/boto/cloudfront/identity.py b/backup/src/boto/cloudfront/identity.py deleted file mode 100644 index 1571e87..0000000 --- a/backup/src/boto/cloudfront/identity.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import uuid - -class OriginAccessIdentity: - - def __init__(self, connection=None, config=None, id='', - s3_user_id='', comment=''): - self.connection = connection - self.config = config - self.id = id - self.s3_user_id = s3_user_id - self.comment = comment - self.etag = None - - def startElement(self, name, attrs, connection): - if name == 'CloudFrontOriginAccessIdentityConfig': - self.config = OriginAccessIdentityConfig() - return self.config - else: - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'S3CanonicalUserId': - self.s3_user_id = value - elif name == 'Comment': - self.comment = value - else: - setattr(self, name, value) - - def update(self, comment=None): - new_config = OriginAccessIdentityConfig(self.connection, - self.config.caller_reference, - self.config.comment) - if comment != None: - new_config.comment = comment - self.etag = self.connection.set_origin_identity_config(self.id, self.etag, new_config) - self.config = new_config - - def delete(self): - return self.connection.delete_origin_access_identity(self.id, self.etag) - - def uri(self): - return 'origin-access-identity/cloudfront/%s' % self.id - -class OriginAccessIdentityConfig: - - def __init__(self, connection=None, caller_reference='', comment=''): - self.connection = connection - if caller_reference: - self.caller_reference = caller_reference - else: - self.caller_reference = str(uuid.uuid4()) - self.comment = comment - - def to_xml(self): - s = '\n' - s += '\n' - s += ' %s\n' % self.caller_reference - if self.comment: - s += ' %s\n' % self.comment - s += '\n' - return s - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Comment': - self.comment = value - elif name == 'CallerReference': - self.caller_reference = value - else: - setattr(self, name, value) - -class OriginAccessIdentitySummary: - - def __init__(self, connection=None, id='', - s3_user_id='', comment=''): - self.connection = connection - self.id = id - self.s3_user_id = s3_user_id - self.comment = comment - self.etag = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'S3CanonicalUserId': - self.s3_user_id = value - elif name == 'Comment': - self.comment = value - else: - setattr(self, name, value) - - def get_origin_access_identity(self): - return self.connection.get_origin_access_identity_info(self.id) - diff --git a/backup/src/boto/cloudfront/invalidation.py b/backup/src/boto/cloudfront/invalidation.py deleted file mode 100644 index ea13a67..0000000 --- a/backup/src/boto/cloudfront/invalidation.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2006-2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import uuid -import urllib - -class InvalidationBatch(object): - """A simple invalidation request. - :see: http://docs.amazonwebservices.com/AmazonCloudFront/2010-08-01/APIReference/index.html?InvalidationBatchDatatype.html - """ - - def __init__(self, paths=[], connection=None, distribution=None, caller_reference=''): - """Create a new invalidation request: - :paths: An array of paths to invalidate - """ - self.paths = paths - self.distribution = distribution - self.caller_reference = caller_reference - if not self.caller_reference: - self.caller_reference = str(uuid.uuid4()) - - # If we passed in a distribution, - # then we use that as the connection object - if distribution: - self.connection = connection - else: - self.connection = connection - - def add(self, path): - """Add another path to this invalidation request""" - return self.paths.append(path) - - def remove(self, path): - """Remove a path from this invalidation request""" - return self.paths.remove(path) - - def __iter__(self): - return iter(self.paths) - - def __getitem__(self, i): - return self.paths[i] - - def __setitem__(self, k, v): - self.paths[k] = v - - def escape(self, p): - """Escape a path, make sure it begins with a slash and contains no invalid characters""" - if not p[0] == "/": - p = "/%s" % p - return urllib.quote(p) - - def to_xml(self): - """Get this batch as XML""" - assert self.connection != None - s = '\n' - s += '\n' % self.connection.Version - for p in self.paths: - s += ' %s\n' % self.escape(p) - s += ' %s\n' % self.caller_reference - s += '\n' - return s - - def startElement(self, name, attrs, connection): - if name == "InvalidationBatch": - self.paths = [] - return None - - def endElement(self, name, value, connection): - if name == 'Path': - self.paths.append(value) - elif name == "Status": - self.status = value - elif name == "Id": - self.id = value - elif name == "CreateTime": - self.create_time = value - elif name == "CallerReference": - self.caller_reference = value - return None diff --git a/backup/src/boto/cloudfront/logging.py b/backup/src/boto/cloudfront/logging.py deleted file mode 100644 index 6c2f4fd..0000000 --- a/backup/src/boto/cloudfront/logging.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class LoggingInfo(object): - - def __init__(self, bucket='', prefix=''): - self.bucket = bucket - self.prefix = prefix - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Bucket': - self.bucket = value - elif name == 'Prefix': - self.prefix = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/cloudfront/object.py b/backup/src/boto/cloudfront/object.py deleted file mode 100644 index 3574d13..0000000 --- a/backup/src/boto/cloudfront/object.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.s3.key import Key - -class Object(Key): - - def __init__(self, bucket, name=None): - Key.__init__(self, bucket, name=name) - self.distribution = bucket.distribution - - def __repr__(self): - return '' % (self.distribution.config.origin, self.name) - - def url(self, scheme='http'): - url = '%s://' % scheme - url += self.distribution.domain_name - if scheme.lower().startswith('rtmp'): - url += '/cfx/st/' - else: - url += '/' - url += self.name - return url - -class StreamingObject(Object): - - def url(self, scheme='rtmp'): - return Object.url(self, scheme) - - diff --git a/backup/src/boto/cloudfront/origin.py b/backup/src/boto/cloudfront/origin.py deleted file mode 100644 index 57af846..0000000 --- a/backup/src/boto/cloudfront/origin.py +++ /dev/null @@ -1,150 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from identity import OriginAccessIdentity - -def get_oai_value(origin_access_identity): - if isinstance(origin_access_identity, OriginAccessIdentity): - return origin_access_identity.uri() - else: - return origin_access_identity - -class S3Origin(object): - """ - Origin information to associate with the distribution. - If your distribution will use an Amazon S3 origin, - then you use the S3Origin element. - """ - - def __init__(self, dns_name=None, origin_access_identity=None): - """ - :param dns_name: The DNS name of your Amazon S3 bucket to - associate with the distribution. - For example: mybucket.s3.amazonaws.com. - :type dns_name: str - - :param origin_access_identity: The CloudFront origin access - identity to associate with the - distribution. If you want the - distribution to serve private content, - include this element; if you want the - distribution to serve public content, - remove this element. - :type origin_access_identity: str - - """ - self.dns_name = dns_name - self.origin_access_identity = origin_access_identity - - def __repr__(self): - return '' % self.dns_name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'DNSName': - self.dns_name = value - elif name == 'OriginAccessIdentity': - self.origin_access_identity = value - else: - setattr(self, name, value) - - def to_xml(self): - s = ' \n' - s += ' %s\n' % self.dns_name - if self.origin_access_identity: - val = get_oai_value(self.origin_access_identity) - s += ' %s\n' % val - s += ' \n' - return s - -class CustomOrigin(object): - """ - Origin information to associate with the distribution. - If your distribution will use a non-Amazon S3 origin, - then you use the CustomOrigin element. - """ - - def __init__(self, dns_name=None, http_port=80, https_port=443, - origin_protocol_policy=None): - """ - :param dns_name: The DNS name of your Amazon S3 bucket to - associate with the distribution. - For example: mybucket.s3.amazonaws.com. - :type dns_name: str - - :param http_port: The HTTP port the custom origin listens on. - :type http_port: int - - :param https_port: The HTTPS port the custom origin listens on. - :type http_port: int - - :param origin_protocol_policy: The origin protocol policy to - apply to your origin. If you - specify http-only, CloudFront - will use HTTP only to access the origin. - If you specify match-viewer, CloudFront - will fetch from your origin using HTTP - or HTTPS, based on the protocol of the - viewer request. - :type origin_protocol_policy: str - - """ - self.dns_name = dns_name - self.http_port = http_port - self.https_port = https_port - self.origin_protocol_policy = origin_protocol_policy - - def __repr__(self): - return '' % self.dns_name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'DNSName': - self.dns_name = value - elif name == 'HTTPPort': - try: - self.http_port = int(value) - except ValueError: - self.http_port = value - elif name == 'HTTPSPort': - try: - self.https_port = int(value) - except ValueError: - self.https_port = value - elif name == 'OriginProtocolPolicy': - self.origin_protocol_policy = value - else: - setattr(self, name, value) - - def to_xml(self): - s = ' \n' - s += ' %s\n' % self.dns_name - s += ' %d\n' % self.http_port - s += ' %d\n' % self.https_port - s += ' %s\n' % self.origin_protocol_policy - s += ' \n' - return s - diff --git a/backup/src/boto/cloudfront/signers.py b/backup/src/boto/cloudfront/signers.py deleted file mode 100644 index 0b0cd50..0000000 --- a/backup/src/boto/cloudfront/signers.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Signer: - - def __init__(self): - self.id = None - self.key_pair_ids = [] - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Self': - self.id = 'Self' - elif name == 'AwsAccountNumber': - self.id = value - elif name == 'KeyPairId': - self.key_pair_ids.append(value) - -class ActiveTrustedSigners(list): - - def startElement(self, name, attrs, connection): - if name == 'Signer': - s = Signer() - self.append(s) - return s - - def endElement(self, name, value, connection): - pass - -class TrustedSigners(list): - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Self': - self.append(name) - elif name == 'AwsAccountNumber': - self.append(value) - diff --git a/backup/src/boto/connection.py b/backup/src/boto/connection.py deleted file mode 100644 index 76e9ffe..0000000 --- a/backup/src/boto/connection.py +++ /dev/null @@ -1,637 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010 Google -# Copyright (c) 2008 rPath, Inc. -# Copyright (c) 2009 The Echo Nest Corporation -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. - -""" -Handles basic connections to AWS -""" - -import base64 -import errno -import httplib -import os -import Queue -import re -import socket -import sys -import time -import urllib, urlparse -import xml.sax - -import auth -import auth_handler -import boto -import boto.utils - -from boto import config, UserAgent, handler -from boto.exception import AWSConnectionError, BotoClientError, BotoServerError -from boto.provider import Provider -from boto.resultset import ResultSet - - -PORTS_BY_SECURITY = { True: 443, False: 80 } - -class ConnectionPool: - def __init__(self, hosts, connections_per_host): - self._hosts = boto.utils.LRUCache(hosts) - self.connections_per_host = connections_per_host - - def __getitem__(self, key): - if key not in self._hosts: - self._hosts[key] = Queue.Queue(self.connections_per_host) - return self._hosts[key] - - def __repr__(self): - return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) - -class HTTPRequest(object): - - def __init__(self, method, protocol, host, port, path, auth_path, - params, headers, body): - """Represents an HTTP request. - - :type method: string - :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. - - :type protocol: string - :param protocol: The http protocol used, 'http' or 'https'. - - :type host: string - :param host: Host to which the request is addressed. eg. abc.com - - :type port: int - :param port: port on which the request is being sent. Zero means unset, - in which case default port will be chosen. - - :type path: string - :param path: URL path that is bein accessed. - - :type auth_path: string - :param path: The part of the URL path used when creating the - authentication string. - - :type params: dict - :param params: HTTP url query parameters, with key as name of the param, - and value as value of param. - - :type headers: dict - :param headers: HTTP headers, with key as name of the header and value - as value of header. - - :type body: string - :param body: Body of the HTTP request. If not present, will be None or - empty string (''). - """ - self.method = method - self.protocol = protocol - self.host = host - self.port = port - self.path = path - self.auth_path = auth_path - self.params = params - self.headers = headers - self.body = body - - def __str__(self): - return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' - 'params(%s) headers(%s) body(%s)') % (self.method, - self.protocol, self.host, self.port, self.path, self.params, - self.headers, self.body)) - -class AWSAuthConnection(object): - def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, path='/', provider='aws'): - """ - :type host: str - :param host: The host to make the connection to - - :keyword str aws_access_key_id: Your AWS Access Key ID (provided by - Amazon). If none is specified, the value in your - ``AWS_ACCESS_KEY_ID`` environmental variable is used. - :keyword str aws_secret_access_key: Your AWS Secret Access Key - (provided by Amazon). If none is specified, the value in your - ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. - - :type is_secure: boolean - :param is_secure: Whether the connection is over SSL - - :type https_connection_factory: list or tuple - :param https_connection_factory: A pair of an HTTP connection - factory and the exceptions to catch. - The factory should have a similar - interface to L{httplib.HTTPSConnection}. - - :param str proxy: Address/hostname for a proxy server - - :type proxy_port: int - :param proxy_port: The port to use when connecting over a proxy - - :type proxy_user: str - :param proxy_user: The username to connect with on the proxy - - :type proxy_pass: str - :param proxy_pass: The password to use when connection over a proxy. - - :type port: int - :param port: The port to use to connect - """ - self.num_retries = 5 - # Override passed-in is_secure setting if value was defined in config. - if config.has_option('Boto', 'is_secure'): - is_secure = config.getboolean('Boto', 'is_secure') - self.is_secure = is_secure - self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) - # define exceptions from httplib that we want to catch and retry - self.http_exceptions = (httplib.HTTPException, socket.error, - socket.gaierror) - # define values in socket exceptions we don't want to catch - self.socket_exception_values = (errno.EINTR,) - if https_connection_factory is not None: - self.https_connection_factory = https_connection_factory[0] - self.http_exceptions += https_connection_factory[1] - else: - self.https_connection_factory = None - if (is_secure): - self.protocol = 'https' - else: - self.protocol = 'http' - self.host = host - self.path = path - if debug: - self.debug = debug - else: - self.debug = config.getint('Boto', 'debug', debug) - if port: - self.port = port - else: - self.port = PORTS_BY_SECURITY[is_secure] - - self.provider = Provider(provider, - aws_access_key_id, - aws_secret_access_key) - - # allow config file to override default host - if self.provider.host: - self.host = self.provider.host - - # cache up to 20 connections per host, up to 20 hosts - self._pool = ConnectionPool(20, 20) - self._connection = (self.server_name(), self.is_secure) - self._last_rs = None - self._auth_handler = auth.get_auth_handler( - host, config, self.provider, self._required_auth_capability()) - - def __repr__(self): - return '%s:%s' % (self.__class__.__name__, self.host) - - def _required_auth_capability(self): - return [] - - def _cached_name(self, host, is_secure): - if host is None: - host = self.server_name() - cached_name = is_secure and 'https://' or 'http://' - cached_name += host - return cached_name - - def connection(self): - return self.get_http_connection(*self._connection) - connection = property(connection) - - def aws_access_key_id(self): - return self.provider.access_key - aws_access_key_id = property(aws_access_key_id) - gs_access_key_id = aws_access_key_id - access_key = aws_access_key_id - - def aws_secret_access_key(self): - return self.provider.secret_key - aws_secret_access_key = property(aws_secret_access_key) - gs_secret_access_key = aws_secret_access_key - secret_key = aws_secret_access_key - - def get_path(self, path='/'): - pos = path.find('?') - if pos >= 0: - params = path[pos:] - path = path[:pos] - else: - params = None - if path[-1] == '/': - need_trailing = True - else: - need_trailing = False - path_elements = self.path.split('/') - path_elements.extend(path.split('/')) - path_elements = [p for p in path_elements if p] - path = '/' + '/'.join(path_elements) - if path[-1] != '/' and need_trailing: - path += '/' - if params: - path = path + params - return path - - def server_name(self, port=None): - if not port: - port = self.port - if port == 80: - signature_host = self.host - else: - # This unfortunate little hack can be attributed to - # a difference in the 2.6 version of httplib. In old - # versions, it would append ":443" to the hostname sent - # in the Host header and so we needed to make sure we - # did the same when calculating the V2 signature. In 2.6 - # (and higher!) - # it no longer does that. Hence, this kludge. - if sys.version[:3] in ('2.6', '2.7') and port == 443: - signature_host = self.host - else: - signature_host = '%s:%d' % (self.host, port) - return signature_host - - def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): - self.proxy = proxy - self.proxy_port = proxy_port - self.proxy_user = proxy_user - self.proxy_pass = proxy_pass - if os.environ.has_key('http_proxy') and not self.proxy: - pattern = re.compile( - '(?:http://)?' \ - '(?:(?P\w+):(?P.*)@)?' \ - '(?P[\w\-\.]+)' \ - '(?::(?P\d+))?' - ) - match = pattern.match(os.environ['http_proxy']) - if match: - self.proxy = match.group('host') - self.proxy_port = match.group('port') - self.proxy_user = match.group('user') - self.proxy_pass = match.group('pass') - else: - if not self.proxy: - self.proxy = config.get_value('Boto', 'proxy', None) - if not self.proxy_port: - self.proxy_port = config.get_value('Boto', 'proxy_port', None) - if not self.proxy_user: - self.proxy_user = config.get_value('Boto', 'proxy_user', None) - if not self.proxy_pass: - self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) - - if not self.proxy_port and self.proxy: - print "http_proxy environment variable does not specify " \ - "a port, using default" - self.proxy_port = self.port - self.use_proxy = (self.proxy != None) - - def get_http_connection(self, host, is_secure): - queue = self._pool[self._cached_name(host, is_secure)] - try: - return queue.get_nowait() - except Queue.Empty: - return self.new_http_connection(host, is_secure) - - def new_http_connection(self, host, is_secure): - if self.use_proxy: - host = '%s:%d' % (self.proxy, int(self.proxy_port)) - if host is None: - host = self.server_name() - if is_secure: - boto.log.debug('establishing HTTPS connection') - if self.use_proxy: - connection = self.proxy_ssl() - elif self.https_connection_factory: - connection = self.https_connection_factory(host) - else: - connection = httplib.HTTPSConnection(host) - else: - boto.log.debug('establishing HTTP connection') - connection = httplib.HTTPConnection(host) - if self.debug > 1: - connection.set_debuglevel(self.debug) - # self.connection must be maintained for backwards-compatibility - # however, it must be dynamically pulled from the connection pool - # set a private variable which will enable that - if host.split(':')[0] == self.host and is_secure == self.is_secure: - self._connection = (host, is_secure) - return connection - - def put_http_connection(self, host, is_secure, connection): - try: - self._pool[self._cached_name(host, is_secure)].put_nowait(connection) - except Queue.Full: - # gracefully fail in case of pool overflow - connection.close() - - def proxy_ssl(self): - host = '%s:%d' % (self.host, self.port) - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - try: - sock.connect((self.proxy, int(self.proxy_port))) - except: - raise - sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) - sock.sendall("User-Agent: %s\r\n" % UserAgent) - if self.proxy_user and self.proxy_pass: - for k, v in self.get_proxy_auth_header().items(): - sock.sendall("%s: %s\r\n" % (k, v)) - sock.sendall("\r\n") - resp = httplib.HTTPResponse(sock, strict=True) - resp.begin() - - if resp.status != 200: - # Fake a socket error, use a code that make it obvious it hasn't - # been generated by the socket library - raise socket.error(-71, - "Error talking to HTTP proxy %s:%s: %s (%s)" % - (self.proxy, self.proxy_port, resp.status, resp.reason)) - - # We can safely close the response, it duped the original socket - resp.close() - - h = httplib.HTTPConnection(host) - - # Wrap the socket in an SSL socket - if hasattr(httplib, 'ssl'): - sslSock = httplib.ssl.SSLSocket(sock) - else: # Old Python, no ssl module - sslSock = socket.ssl(sock, None, None) - sslSock = httplib.FakeSocket(sock, sslSock) - # This is a bit unclean - h.sock = sslSock - return h - - def prefix_proxy_to_path(self, path, host=None): - path = self.protocol + '://' + (host or self.server_name()) + path - return path - - def get_proxy_auth_header(self): - auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) - return {'Proxy-Authorization': 'Basic %s' % auth} - - def _mexe(self, method, path, data, headers, host=None, sender=None, - override_num_retries=None): - """ - mexe - Multi-execute inside a loop, retrying multiple times to handle - transient Internet errors by simply trying again. - Also handles redirects. - - This code was inspired by the S3Utils classes posted to the boto-users - Google group by Larry Bates. Thanks! - """ - boto.log.debug('Method: %s' % method) - boto.log.debug('Path: %s' % path) - boto.log.debug('Data: %s' % data) - boto.log.debug('Headers: %s' % headers) - boto.log.debug('Host: %s' % host) - response = None - body = None - e = None - if override_num_retries is None: - num_retries = config.getint('Boto', 'num_retries', self.num_retries) - else: - num_retries = override_num_retries - i = 0 - connection = self.get_http_connection(host, self.is_secure) - while i <= num_retries: - try: - if callable(sender): - response = sender(connection, method, path, data, headers) - else: - connection.request(method, path, data, headers) - response = connection.getresponse() - location = response.getheader('location') - # -- gross hack -- - # httplib gets confused with chunked responses to HEAD requests - # so I have to fake it out - if method == 'HEAD' and getattr(response, 'chunked', False): - response.chunked = 0 - if response.status == 500 or response.status == 503: - boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2 ** i)) - body = response.read() - elif response.status == 408: - body = response.read() - print '-------------------------' - print ' 4 0 8 ' - print 'path=%s' % path - print body - print '-------------------------' - elif response.status < 300 or response.status >= 400 or \ - not location: - self.put_http_connection(host, self.is_secure, connection) - return response - else: - scheme, host, path, params, query, fragment = \ - urlparse.urlparse(location) - if query: - path += '?' + query - boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) - connection = self.get_http_connection(host, scheme == 'https') - continue - except KeyboardInterrupt: - sys.exit('Keyboard Interrupt') - except self.http_exceptions, e: - boto.log.debug('encountered %s exception, reconnecting' % \ - e.__class__.__name__) - connection = self.new_http_connection(host, self.is_secure) - time.sleep(2 ** i) - i += 1 - # If we made it here, it's because we have exhausted our retries and stil haven't - # succeeded. So, if we have a response object, use it to raise an exception. - # Otherwise, raise the exception that must have already happened. - if response: - raise BotoServerError(response.status, response.reason, body) - elif e: - raise e - else: - raise BotoClientError('Please report this exception as a Boto Issue!') - - def build_base_http_request(self, method, path, auth_path, - params=None, headers=None, data='', host=None): - path = self.get_path(path) - if auth_path is not None: - auth_path = self.get_path(auth_path) - if params == None: - params = {} - else: - params = params.copy() - if headers == None: - headers = {} - else: - headers = headers.copy() - host = host or self.host - if self.use_proxy: - path = self.prefix_proxy_to_path(path, host) - if self.proxy_user and self.proxy_pass and not self.is_secure: - # If is_secure, we don't have to set the proxy authentication - # header here, we did that in the CONNECT to the proxy. - headers.update(self.get_proxy_auth_header()) - return HTTPRequest(method, self.protocol, host, self.port, - path, auth_path, params, headers, data) - - def fill_in_auth(self, http_request, **kwargs): - headers = http_request.headers - for key in headers: - val = headers[key] - if isinstance(val, unicode): - headers[key] = urllib.quote_plus(val.encode('utf-8')) - - self._auth_handler.add_auth(http_request, **kwargs) - - headers['User-Agent'] = UserAgent - if not headers.has_key('Content-Length'): - headers['Content-Length'] = str(len(http_request.body)) - return http_request - - def _send_http_request(self, http_request, sender=None, - override_num_retries=None): - return self._mexe(http_request.method, http_request.path, - http_request.body, http_request.headers, - http_request.host, sender, override_num_retries) - - def make_request(self, method, path, headers=None, data='', host=None, - auth_path=None, sender=None, override_num_retries=None): - """Makes a request to the server, with stock multiple-retry logic.""" - http_request = self.build_base_http_request(method, path, auth_path, - {}, headers, data, host) - http_request = self.fill_in_auth(http_request) - return self._send_http_request(http_request, sender, - override_num_retries) - - def close(self): - """(Optional) Close any open HTTP connections. This is non-destructive, - and making a new request will open a connection again.""" - - boto.log.debug('closing all HTTP connections') - self.connection = None # compat field - -class AWSQueryConnection(AWSAuthConnection): - - APIVersion = '' - ResponseError = BotoServerError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, host=None, debug=0, - https_connection_factory=None, path='/'): - AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - debug, https_connection_factory, path) - - def _required_auth_capability(self): - return [] - - def get_utf8_value(self, value): - return boto.utils.get_utf8_value(value) - - def make_request(self, action, params=None, path='/', verb='GET'): - http_request = self.build_base_http_request(verb, path, None, - params, {}, '', - self.server_name()) - if action: - http_request.params['Action'] = action - http_request.params['Version'] = self.APIVersion - http_request = self.fill_in_auth(http_request) - return self._send_http_request(http_request) - - def build_list_params(self, params, items, label): - if isinstance(items, str): - items = [items] - for i in range(1, len(items) + 1): - params['%s.%d' % (label, i)] = items[i - 1] - - # generics - - def get_list(self, action, params, markers, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - rs = ResultSet(markers) - h = handler.XmlHandler(rs, parent) - xml.sax.parseString(body, h) - return rs - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_object(self, action, params, cls, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - obj = cls(parent) - h = handler.XmlHandler(obj, parent) - xml.sax.parseString(body, h) - return obj - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_status(self, action, params, path='/', parent=None, verb='GET'): - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if not body: - boto.log.error('Null body %s' % body) - raise self.ResponseError(response.status, response.reason, body) - elif response.status == 200: - rs = ResultSet() - h = handler.XmlHandler(rs, parent) - xml.sax.parseString(body, h) - return rs.status - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) diff --git a/backup/src/boto/contrib/__init__.py b/backup/src/boto/contrib/__init__.py deleted file mode 100644 index 303dbb6..0000000 --- a/backup/src/boto/contrib/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/backup/src/boto/contrib/m2helpers.py b/backup/src/boto/contrib/m2helpers.py deleted file mode 100644 index 82d2730..0000000 --- a/backup/src/boto/contrib/m2helpers.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2006,2007 Jon Colverson -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -This module was contributed by Jon Colverson. It provides a couple of helper -functions that allow you to use M2Crypto's implementation of HTTPSConnection -rather than the default version in httplib.py. The main benefit is that -M2Crypto's version verifies the certificate of the server. - -To use this feature, do something like this: - -from boto.ec2.connection import EC2Connection - -ec2 = EC2Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY, - https_connection_factory=https_connection_factory(cafile=CA_FILE)) - -See http://code.google.com/p/boto/issues/detail?id=57 for more details. -""" -from M2Crypto import SSL -from M2Crypto.httpslib import HTTPSConnection - -def secure_context(cafile=None, capath=None): - ctx = SSL.Context() - ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) - if ctx.load_verify_locations(cafile=cafile, capath=capath) != 1: - raise Exception("Couldn't load certificates") - return ctx - -def https_connection_factory(cafile=None, capath=None): - def factory(*args, **kwargs): - return HTTPSConnection( - ssl_context=secure_context(cafile=cafile, capath=capath), - *args, **kwargs) - return (factory, (SSL.SSLError,)) diff --git a/backup/src/boto/contrib/ymlmessage.py b/backup/src/boto/contrib/ymlmessage.py deleted file mode 100644 index b9a2c93..0000000 --- a/backup/src/boto/contrib/ymlmessage.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2006,2007 Chris Moyer -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -This module was contributed by Chris Moyer. It provides a subclass of the -SQS Message class that supports YAML as the body of the message. - -This module requires the yaml module. -""" -from boto.sqs.message import Message -import yaml - -class YAMLMessage(Message): - """ - The YAMLMessage class provides a YAML compatible message. Encoding and - decoding are handled automaticaly. - - Access this message data like such: - - m.data = [ 1, 2, 3] - m.data[0] # Returns 1 - - This depends on the PyYAML package - """ - - def __init__(self, queue=None, body='', xml_attrs=None): - self.data = None - Message.__init__(self, queue, body) - - def set_body(self, body): - self.data = yaml.load(body) - - def get_body(self): - return yaml.dump(self.data) diff --git a/backup/src/boto/ec2/__init__.py b/backup/src/boto/ec2/__init__.py deleted file mode 100644 index 8bb3f53..0000000 --- a/backup/src/boto/ec2/__init__.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -""" -This module provides an interface to the Elastic Compute Cloud (EC2) -service from AWS. -""" -from boto.ec2.connection import EC2Connection - -def regions(**kw_params): - """ - Get all available regions for the EC2 service. - You may pass any of the arguments accepted by the EC2Connection - object's constructor as keyword arguments and they will be - passed along to the EC2Connection object. - - :rtype: list - :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` - """ - c = EC2Connection(**kw_params) - return c.get_all_regions() - -def connect_to_region(region_name, **kw_params): - for region in regions(**kw_params): - if region.name == region_name: - return region.connect(**kw_params) - return None - -def get_region(region_name, **kw_params): - for region in regions(**kw_params): - if region.name == region_name: - return region - return None - diff --git a/backup/src/boto/ec2/address.py b/backup/src/boto/ec2/address.py deleted file mode 100644 index 60ed406..0000000 --- a/backup/src/boto/ec2/address.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Elastic IP Address -""" - -from boto.ec2.ec2object import EC2Object - -class Address(EC2Object): - - def __init__(self, connection=None, public_ip=None, instance_id=None): - EC2Object.__init__(self, connection) - self.connection = connection - self.public_ip = public_ip - self.instance_id = instance_id - - def __repr__(self): - return 'Address:%s' % self.public_ip - - def endElement(self, name, value, connection): - if name == 'publicIp': - self.public_ip = value - elif name == 'instanceId': - self.instance_id = value - else: - setattr(self, name, value) - - def release(self): - return self.connection.release_address(self.public_ip) - - delete = release - - def associate(self, instance_id): - return self.connection.associate_address(instance_id, self.public_ip) - - def disassociate(self): - return self.connection.disassociate_address(self.public_ip) - - diff --git a/backup/src/boto/ec2/autoscale/__init__.py b/backup/src/boto/ec2/autoscale/__init__.py deleted file mode 100644 index e8dd695..0000000 --- a/backup/src/boto/ec2/autoscale/__init__.py +++ /dev/null @@ -1,282 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -This module provides an interface to the Elastic Compute Cloud (EC2) -Auto Scaling service. -""" - -import base64 -import boto -from boto.connection import AWSQueryConnection -from boto.ec2.regioninfo import RegionInfo -from boto.ec2.autoscale.request import Request -from boto.ec2.autoscale.trigger import Trigger -from boto.ec2.autoscale.launchconfig import LaunchConfiguration -from boto.ec2.autoscale.group import AutoScalingGroup -from boto.ec2.autoscale.activity import Activity - -RegionData = { - 'us-east-1' : 'autoscaling.us-east-1.amazonaws.com', - 'us-west-1' : 'autoscaling.us-west-1.amazonaws.com', - 'eu-west-1' : 'autoscaling.eu-west-1.amazonaws.com', - 'ap-southeast-1' : 'autoscaling.ap-southeast-1.amazonaws.com'} - -def regions(): - """ - Get all available regions for the Auto Scaling service. - - :rtype: list - :return: A list of :class:`boto.RegionInfo` instances - """ - regions = [] - for region_name in RegionData: - region = RegionInfo(name=region_name, - endpoint=RegionData[region_name], - connection_cls=AutoScaleConnection) - regions.append(region) - return regions - -def connect_to_region(region_name, **kw_params): - """ - Given a valid region name, return a - :class:`boto.ec2.autoscale.AutoScaleConnection`. - - :param str region_name: The name of the region to connect to. - - :rtype: :class:`boto.ec2.AutoScaleConnection` or ``None`` - :return: A connection to the given region, or None if an invalid region - name is given - """ - for region in regions(): - if region.name == region_name: - return region.connect(**kw_params) - return None - - -class AutoScaleConnection(AWSQueryConnection): - APIVersion = boto.config.get('Boto', 'autoscale_version', '2010-08-01') - Endpoint = boto.config.get('Boto', 'autoscale_endpoint', - 'autoscaling.amazonaws.com') - DefaultRegionName = 'us-east-1' - DefaultRegionEndpoint = 'autoscaling.amazonaws.com' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=1, - https_connection_factory=None, region=None, path='/'): - """ - Init method to create a new connection to the AutoScaling service. - - B{Note:} The host argument is overridden by the host specified in the - boto configuration file. - """ - if not region: - region = RegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint, - AutoScaleConnection) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path=path) - - def _required_auth_capability(self): - return ['ec2'] - - def build_list_params(self, params, items, label): - """ items is a list of dictionaries or strings: - [{'Protocol' : 'HTTP', - 'LoadBalancerPort' : '80', - 'InstancePort' : '80'},..] etc. - or - ['us-east-1b',...] - """ - # different from EC2 list params - for i in xrange(1, len(items)+1): - if isinstance(items[i-1], dict): - for k, v in items[i-1].iteritems(): - params['%s.member.%d.%s' % (label, i, k)] = v - elif isinstance(items[i-1], basestring): - params['%s.member.%d' % (label, i)] = items[i-1] - - def _update_group(self, op, as_group): - params = { - 'AutoScalingGroupName' : as_group.name, - 'Cooldown' : as_group.cooldown, - 'LaunchConfigurationName' : as_group.launch_config_name, - 'MinSize' : as_group.min_size, - 'MaxSize' : as_group.max_size, - } - if op.startswith('Create'): - if as_group.availability_zones: - zones = as_group.availability_zones - else: - zones = [as_group.availability_zone] - self.build_list_params(params, as_group.load_balancers, - 'LoadBalancerNames') - self.build_list_params(params, zones, - 'AvailabilityZones') - return self.get_object(op, params, Request) - - def create_auto_scaling_group(self, as_group): - """ - Create auto scaling group. - """ - return self._update_group('CreateAutoScalingGroup', as_group) - - def create_launch_configuration(self, launch_config): - """ - Creates a new Launch Configuration. - - :type launch_config: boto.ec2.autoscale.launchconfig.LaunchConfiguration - :param launch_config: LaunchConfiguraiton object. - - """ - params = { - 'ImageId' : launch_config.image_id, - 'KeyName' : launch_config.key_name, - 'LaunchConfigurationName' : launch_config.name, - 'InstanceType' : launch_config.instance_type, - } - if launch_config.user_data: - params['UserData'] = base64.b64encode(launch_config.user_data) - if launch_config.kernel_id: - params['KernelId'] = launch_config.kernel_id - if launch_config.ramdisk_id: - params['RamdiskId'] = launch_config.ramdisk_id - if launch_config.block_device_mappings: - self.build_list_params(params, launch_config.block_device_mappings, - 'BlockDeviceMappings') - self.build_list_params(params, launch_config.security_groups, - 'SecurityGroups') - return self.get_object('CreateLaunchConfiguration', params, - Request, verb='POST') - - def create_trigger(self, trigger): - """ - - """ - params = {'TriggerName' : trigger.name, - 'AutoScalingGroupName' : trigger.autoscale_group.name, - 'MeasureName' : trigger.measure_name, - 'Statistic' : trigger.statistic, - 'Period' : trigger.period, - 'Unit' : trigger.unit, - 'LowerThreshold' : trigger.lower_threshold, - 'LowerBreachScaleIncrement' : trigger.lower_breach_scale_increment, - 'UpperThreshold' : trigger.upper_threshold, - 'UpperBreachScaleIncrement' : trigger.upper_breach_scale_increment, - 'BreachDuration' : trigger.breach_duration} - # dimensions should be a list of tuples - dimensions = [] - for dim in trigger.dimensions: - name, value = dim - dimensions.append(dict(Name=name, Value=value)) - self.build_list_params(params, dimensions, 'Dimensions') - - req = self.get_object('CreateOrUpdateScalingTrigger', params, - Request) - return req - - def get_all_groups(self, names=None): - """ - """ - params = {} - if names: - self.build_list_params(params, names, 'AutoScalingGroupNames') - return self.get_list('DescribeAutoScalingGroups', params, - [('member', AutoScalingGroup)]) - - def get_all_launch_configurations(self, names=None): - """ - """ - params = {} - if names: - self.build_list_params(params, names, 'LaunchConfigurationNames') - return self.get_list('DescribeLaunchConfigurations', params, - [('member', LaunchConfiguration)]) - - def get_all_activities(self, autoscale_group, - activity_ids=None, - max_records=100): - """ - Get all activities for the given autoscaling group. - - :type autoscale_group: str or AutoScalingGroup object - :param autoscale_group: The auto scaling group to get activities on. - - @max_records: int - :param max_records: Maximum amount of activities to return. - """ - name = autoscale_group - if isinstance(autoscale_group, AutoScalingGroup): - name = autoscale_group.name - params = {'AutoScalingGroupName' : name} - if activity_ids: - self.build_list_params(params, activity_ids, 'ActivityIds') - return self.get_list('DescribeScalingActivities', params, - [('member', Activity)]) - - def get_all_triggers(self, autoscale_group): - params = {'AutoScalingGroupName' : autoscale_group} - return self.get_list('DescribeTriggers', params, - [('member', Trigger)]) - - def terminate_instance(self, instance_id, decrement_capacity=True): - params = { - 'InstanceId' : instance_id, - 'ShouldDecrementDesiredCapacity' : decrement_capacity - } - return self.get_object('TerminateInstanceInAutoScalingGroup', params, - Activity) - - def set_instance_health(self, instance_id, health_status, - should_respect_grace_period=True): - """ - Explicitly set the health status of an instance. - - :type instance_id: str - :param instance_id: The identifier of the EC2 instance. - - :type health_status: str - :param health_status: The health status of the instance. - "Healthy" means that the instance is - healthy and should remain in service. - "Unhealthy" means that the instance is - unhealthy. Auto Scaling should terminate - and replace it. - - :type should_respect_grace_period: bool - :param should_respect_grace_period: If True, this call should - respect the grace period - associated with the group. - """ - params = {'InstanceId' : instance_id, - 'HealthStatus' : health_status} - if should_respect_grace_period: - params['ShouldRespectGracePeriod'] = 'true' - else: - params['ShouldRespectGracePeriod'] = 'false' - return self.get_status('SetInstanceHealth', params) - diff --git a/backup/src/boto/ec2/autoscale/activity.py b/backup/src/boto/ec2/autoscale/activity.py deleted file mode 100644 index f895d65..0000000 --- a/backup/src/boto/ec2/autoscale/activity.py +++ /dev/null @@ -1,55 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -class Activity(object): - def __init__(self, connection=None): - self.connection = connection - self.start_time = None - self.activity_id = None - self.progress = None - self.status_code = None - self.cause = None - self.description = None - - def __repr__(self): - return 'Activity:%s status:%s progress:%s' % (self.description, - self.status_code, - self.progress) - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'ActivityId': - self.activity_id = value - elif name == 'StartTime': - self.start_time = value - elif name == 'Progress': - self.progress = value - elif name == 'Cause': - self.cause = value - elif name == 'Description': - self.description = value - elif name == 'StatusCode': - self.status_code = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/ec2/autoscale/group.py b/backup/src/boto/ec2/autoscale/group.py deleted file mode 100644 index 9010a72..0000000 --- a/backup/src/boto/ec2/autoscale/group.py +++ /dev/null @@ -1,235 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import weakref -from boto.ec2.elb.listelement import ListElement -from boto.resultset import ResultSet -from boto.ec2.autoscale.trigger import Trigger -from boto.ec2.autoscale.request import Request - - -class Instance(object): - def __init__(self, connection=None): - self.connection = connection - self.instance_id = '' - - def __repr__(self): - return 'Instance:%s' % self.instance_id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'InstanceId': - self.instance_id = value - else: - setattr(self, name, value) - - -class AutoScalingGroup(object): - def __init__(self, connection=None, group_name=None, - availability_zone=None, launch_config=None, - availability_zones=None, - load_balancers=None, cooldown=0, - min_size=None, max_size=None, - group_arn=None, health_check_type=None, - health_check_period=None, suspended=None, - placement_group=None, vpc_zone=None): - """ - Creates a new AutoScalingGroup with the specified name. - - You must not have already used up your entire quota of - AutoScalingGroups in order for this call to be successful. Once the - creation request is completed, the AutoScalingGroup is ready to be - used in other calls. - - :type name: str - :param name: Name of autoscaling group. - - :type availability_zone: str - :param availability_zone: An availability zone. DEPRECATED - use the - availability_zones parameter, which expects - a list of availability zone - strings - - :type availability_zone: list - :param availability_zone: List of availability zones. - - :type launch_config: str - :param launch_config: Name of launch configuration name. - - :type load_balancers: list - :param load_balancers: List of load balancers. - - :type minsize: int - :param minsize: Minimum size of group - - :type maxsize: int - :param maxsize: Maximum size of group - - :type cooldown: int - :param cooldown: Amount of time after a Scaling Activity completes - before any further scaling activities can start. - - :rtype: tuple - :return: Updated healthcheck for the instances. - """ - self.name = group_name - self.connection = connection - self.min_size = min_size - self.max_size = max_size - self.created_time = None - self.cooldown = cooldown - self.launch_config = launch_config - if self.launch_config: - self.launch_config_name = self.launch_config.name - else: - self.launch_config_name = None - self.desired_capacity = None - lbs = load_balancers or [] - self.load_balancers = ListElement(lbs) - zones = availability_zones or [] - self.availability_zone = availability_zone - self.availability_zones = ListElement(zones) - self.group_arn = group_arn - self.health_check_type = health_check_type - self.health_check_period = health_check_period - self.suspended = suspended - self.placement_group = placement_group - self.vpc_zone = vpc_zone - self.metrics = None - self.instances = None - - def __repr__(self): - return 'AutoScalingGroup:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'Instances': - self.instances = ResultSet([('member', Instance)]) - return self.instances - elif name == 'LoadBalancerNames': - return self.load_balancers - elif name == 'AvailabilityZones': - return self.availability_zones - elif name == 'EnabledMetrics': - self.metrics = ResultSet([('member', AutoScalingGroupMetric)]) - return self.metrics - else: - return - - def endElement(self, name, value, connection): - if name == 'MinSize': - self.min_size = value - elif name == 'CreatedTime': - self.created_time = value - elif name == 'DefaultCooldown': - self.cooldown = value - elif name == 'LaunchConfigurationName': - self.launch_config_name = value - elif name == 'DesiredCapacity': - self.desired_capacity = value - elif name == 'MaxSize': - self.max_size = value - elif name == 'AutoScalingGroupName': - self.name = value - elif name == 'AutoScalingGroupARN': - self.group_arn = value - elif name == 'HealthCheckType': - self.health_check_type = value - elif name == 'HealthCheckGracePeriod': - self.health_check_period = value - elif name == 'SuspendedProcesses': - self.suspended = value - elif name == 'PlacementGroup': - self.placement_group = value - elif name == 'VPCZoneIdentifier': - self.vpc_zone = value - else: - setattr(self, name, value) - - def set_capacity(self, capacity): - """ Set the desired capacity for the group. """ - params = { - 'AutoScalingGroupName' : self.name, - 'DesiredCapacity' : capacity, - } - req = self.connection.get_object('SetDesiredCapacity', params, - Request) - self.connection.last_request = req - return req - - def update(self): - """ Sync local changes with AutoScaling group. """ - return self.connection._update_group('UpdateAutoScalingGroup', self) - - def shutdown_instances(self): - """ Convenience method which shuts down all instances associated with - this group. - """ - self.min_size = 0 - self.max_size = 0 - self.update() - - def get_all_triggers(self): - """ Get all triggers for this auto scaling group. """ - params = {'AutoScalingGroupName' : self.name} - triggers = self.connection.get_list('DescribeTriggers', params, - [('member', Trigger)]) - - # allow triggers to be able to access the autoscale group - for tr in triggers: - tr.autoscale_group = weakref.proxy(self) - - return triggers - - def delete(self): - """ Delete this auto-scaling group. """ - params = {'AutoScalingGroupName' : self.name} - return self.connection.get_object('DeleteAutoScalingGroup', params, - Request) - - def get_activities(self, activity_ids=None, max_records=100): - """ - Get all activies for this group. - """ - return self.connection.get_all_activities(self, activity_ids, max_records) - - -class AutoScalingGroupMetric(object): - def __init__(self, connection=None): - - self.connection = connection - self.metric = None - self.granularity = None - - def __repr__(self): - return 'AutoScalingGroupMetric:%s' % self.metric - - def startElement(self, name, attrs, connection): - return - - def endElement(self, name, value, connection): - if name == 'Metric': - self.metric = value - elif name == 'Granularity': - self.granularity = value - else: - setattr(self, name, value) diff --git a/backup/src/boto/ec2/autoscale/instance.py b/backup/src/boto/ec2/autoscale/instance.py deleted file mode 100644 index ffdd5b1..0000000 --- a/backup/src/boto/ec2/autoscale/instance.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -class Instance(object): - def __init__(self, connection=None): - self.connection = connection - self.instance_id = '' - self.lifecycle_state = None - self.availability_zone = '' - - def __repr__(self): - return 'Instance:%s' % self.instance_id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'InstanceId': - self.instance_id = value - elif name == 'LifecycleState': - self.lifecycle_state = value - elif name == 'AvailabilityZone': - self.availability_zone = value - else: - setattr(self, name, value) - - diff --git a/backup/src/boto/ec2/autoscale/launchconfig.py b/backup/src/boto/ec2/autoscale/launchconfig.py deleted file mode 100644 index 7587cb6..0000000 --- a/backup/src/boto/ec2/autoscale/launchconfig.py +++ /dev/null @@ -1,98 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -from boto.ec2.autoscale.request import Request -from boto.ec2.elb.listelement import ListElement - - -class LaunchConfiguration(object): - def __init__(self, connection=None, name=None, image_id=None, - key_name=None, security_groups=None, user_data=None, - instance_type='m1.small', kernel_id=None, - ramdisk_id=None, block_device_mappings=None): - """ - A launch configuration. - - :type name: str - :param name: Name of the launch configuration to create. - - :type image_id: str - :param image_id: Unique ID of the Amazon Machine Image (AMI) which was - assigned during registration. - - :type key_name: str - :param key_name: The name of the EC2 key pair. - - :type security_groups: list - :param security_groups: Names of the security groups with which to - associate the EC2 instances. - - """ - self.connection = connection - self.name = name - self.instance_type = instance_type - self.block_device_mappings = block_device_mappings - self.key_name = key_name - sec_groups = security_groups or [] - self.security_groups = ListElement(sec_groups) - self.image_id = image_id - self.ramdisk_id = ramdisk_id - self.created_time = None - self.kernel_id = kernel_id - self.user_data = user_data - self.created_time = None - - def __repr__(self): - return 'LaunchConfiguration:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'SecurityGroups': - return self.security_groups - else: - return - - def endElement(self, name, value, connection): - if name == 'InstanceType': - self.instance_type = value - elif name == 'LaunchConfigurationName': - self.name = value - elif name == 'KeyName': - self.key_name = value - elif name == 'ImageId': - self.image_id = value - elif name == 'CreatedTime': - self.created_time = value - elif name == 'KernelId': - self.kernel_id = value - elif name == 'RamdiskId': - self.ramdisk_id = value - elif name == 'UserData': - self.user_data = value - else: - setattr(self, name, value) - - def delete(self): - """ Delete this launch configuration. """ - params = {'LaunchConfigurationName' : self.name} - return self.connection.get_object('DeleteLaunchConfiguration', params, - Request) - diff --git a/backup/src/boto/ec2/autoscale/request.py b/backup/src/boto/ec2/autoscale/request.py deleted file mode 100644 index c066dff..0000000 --- a/backup/src/boto/ec2/autoscale/request.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Request(object): - def __init__(self, connection=None): - self.connection = connection - self.request_id = '' - - def __repr__(self): - return 'Request:%s' % self.request_id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'RequestId': - self.request_id = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/ec2/autoscale/trigger.py b/backup/src/boto/ec2/autoscale/trigger.py deleted file mode 100644 index 2840e67..0000000 --- a/backup/src/boto/ec2/autoscale/trigger.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import weakref - -from boto.ec2.autoscale.request import Request - - -class Trigger(object): - """ - An auto scaling trigger. - """ - - def __init__(self, connection=None, name=None, autoscale_group=None, - dimensions=None, measure_name=None, - statistic=None, unit=None, period=60, - lower_threshold=None, - lower_breach_scale_increment=None, - upper_threshold=None, - upper_breach_scale_increment=None, - breach_duration=None): - """ - Initialize an auto-scaling trigger object. - - :type name: str - :param name: The name for this trigger - - :type autoscale_group: str - :param autoscale_group: The name of the AutoScalingGroup that will be - associated with the trigger. The AutoScalingGroup - that will be affected by the trigger when it is - activated. - - :type dimensions: list - :param dimensions: List of tuples, i.e. - ('ImageId', 'i-13lasde') etc. - - :type measure_name: str - :param measure_name: The measure name associated with the metric used by - the trigger to determine when to activate, for - example, CPU, network I/O, or disk I/O. - - :type statistic: str - :param statistic: The particular statistic used by the trigger when - fetching metric statistics to examine. - - :type period: int - :param period: The period associated with the metric statistics in - seconds. Valid Values: 60 or a multiple of 60. - - :type unit: str - :param unit: The unit of measurement. - """ - self.name = name - self.connection = connection - self.dimensions = dimensions - self.breach_duration = breach_duration - self.upper_breach_scale_increment = upper_breach_scale_increment - self.created_time = None - self.upper_threshold = upper_threshold - self.status = None - self.lower_threshold = lower_threshold - self.period = period - self.lower_breach_scale_increment = lower_breach_scale_increment - self.statistic = statistic - self.unit = unit - self.namespace = None - if autoscale_group: - self.autoscale_group = weakref.proxy(autoscale_group) - else: - self.autoscale_group = None - self.measure_name = measure_name - - def __repr__(self): - return 'Trigger:%s' % (self.name) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'BreachDuration': - self.breach_duration = value - elif name == 'TriggerName': - self.name = value - elif name == 'Period': - self.period = value - elif name == 'CreatedTime': - self.created_time = value - elif name == 'Statistic': - self.statistic = value - elif name == 'Unit': - self.unit = value - elif name == 'Namespace': - self.namespace = value - elif name == 'AutoScalingGroupName': - self.autoscale_group_name = value - elif name == 'MeasureName': - self.measure_name = value - else: - setattr(self, name, value) - - def update(self): - """ Write out differences to trigger. """ - self.connection.create_trigger(self) - - def delete(self): - """ Delete this trigger. """ - params = { - 'TriggerName' : self.name, - 'AutoScalingGroupName' : self.autoscale_group_name, - } - req =self.connection.get_object('DeleteTrigger', params, - Request) - self.connection.last_request = req - return req - diff --git a/backup/src/boto/ec2/blockdevicemapping.py b/backup/src/boto/ec2/blockdevicemapping.py deleted file mode 100644 index efbc38b..0000000 --- a/backup/src/boto/ec2/blockdevicemapping.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -class BlockDeviceType(object): - - def __init__(self, connection=None): - self.connection = connection - self.ephemeral_name = None - self.no_device = False - self.volume_id = None - self.snapshot_id = None - self.status = None - self.attach_time = None - self.delete_on_termination = False - self.size = None - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name =='volumeId': - self.volume_id = value - elif name == 'virtualName': - self.ephemeral_name = value - elif name =='NoDevice': - self.no_device = (value == 'true') - elif name =='snapshotId': - self.snapshot_id = value - elif name == 'volumeSize': - self.size = int(value) - elif name == 'status': - self.status = value - elif name == 'attachTime': - self.attach_time = value - elif name == 'deleteOnTermination': - if value == 'true': - self.delete_on_termination = True - else: - self.delete_on_termination = False - else: - setattr(self, name, value) - -# for backwards compatibility -EBSBlockDeviceType = BlockDeviceType - -class BlockDeviceMapping(dict): - - def __init__(self, connection=None): - dict.__init__(self) - self.connection = connection - self.current_name = None - self.current_value = None - - def startElement(self, name, attrs, connection): - if name == 'ebs': - self.current_value = BlockDeviceType(self) - return self.current_value - - def endElement(self, name, value, connection): - if name == 'device' or name == 'deviceName': - self.current_name = value - elif name == 'item': - self[self.current_name] = self.current_value - - def build_list_params(self, params, prefix=''): - i = 1 - for dev_name in self: - pre = '%sBlockDeviceMapping.%d' % (prefix, i) - params['%s.DeviceName' % pre] = dev_name - block_dev = self[dev_name] - if block_dev.ephemeral_name: - params['%s.VirtualName' % pre] = block_dev.ephemeral_name - else: - if block_dev.no_device: - params['%s.Ebs.NoDevice' % pre] = 'true' - if block_dev.snapshot_id: - params['%s.Ebs.SnapshotId' % pre] = block_dev.snapshot_id - if block_dev.size: - params['%s.Ebs.VolumeSize' % pre] = block_dev.size - if block_dev.delete_on_termination: - params['%s.Ebs.DeleteOnTermination' % pre] = 'true' - else: - params['%s.Ebs.DeleteOnTermination' % pre] = 'false' - i += 1 diff --git a/backup/src/boto/ec2/bundleinstance.py b/backup/src/boto/ec2/bundleinstance.py deleted file mode 100644 index 9651992..0000000 --- a/backup/src/boto/ec2/bundleinstance.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Bundle Task -""" - -from boto.ec2.ec2object import EC2Object - -class BundleInstanceTask(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.id = None - self.instance_id = None - self.progress = None - self.start_time = None - self.state = None - self.bucket = None - self.prefix = None - self.upload_policy = None - self.upload_policy_signature = None - self.update_time = None - self.code = None - self.message = None - - def __repr__(self): - return 'BundleInstanceTask:%s' % self.id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'bundleId': - self.id = value - elif name == 'instanceId': - self.instance_id = value - elif name == 'progress': - self.progress = value - elif name == 'startTime': - self.start_time = value - elif name == 'state': - self.state = value - elif name == 'bucket': - self.bucket = value - elif name == 'prefix': - self.prefix = value - elif name == 'uploadPolicy': - self.upload_policy = value - elif name == 'uploadPolicySignature': - self.upload_policy_signature = value - elif name == 'updateTime': - self.update_time = value - elif name == 'code': - self.code = value - elif name == 'message': - self.message = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/ec2/buyreservation.py b/backup/src/boto/ec2/buyreservation.py deleted file mode 100644 index fcd8a77..0000000 --- a/backup/src/boto/ec2/buyreservation.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto.ec2 -from boto.sdb.db.property import StringProperty, IntegerProperty -from boto.manage import propget - -InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', - 'c1.medium', 'c1.xlarge', 'm2.xlarge', - 'm2.2xlarge', 'm2.4xlarge', 'cc1.4xlarge', - 't1.micro'] - -class BuyReservation(object): - - def get_region(self, params): - if not params.get('region', None): - prop = StringProperty(name='region', verbose_name='EC2 Region', - choices=boto.ec2.regions) - params['region'] = propget.get(prop, choices=boto.ec2.regions) - - def get_instance_type(self, params): - if not params.get('instance_type', None): - prop = StringProperty(name='instance_type', verbose_name='Instance Type', - choices=InstanceTypes) - params['instance_type'] = propget.get(prop) - - def get_quantity(self, params): - if not params.get('quantity', None): - prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') - params['quantity'] = propget.get(prop) - - def get_zone(self, params): - if not params.get('zone', None): - prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', - choices=self.ec2.get_all_zones) - params['zone'] = propget.get(prop) - - def get(self, params): - self.get_region(params) - self.ec2 = params['region'].connect() - self.get_instance_type(params) - self.get_zone(params) - self.get_quantity(params) - -if __name__ == "__main__": - obj = BuyReservation() - params = {} - obj.get(params) - offerings = obj.ec2.get_all_reserved_instances_offerings(instance_type=params['instance_type'], - availability_zone=params['zone'].name) - print '\nThe following Reserved Instances Offerings are available:\n' - for offering in offerings: - offering.describe() - prop = StringProperty(name='offering', verbose_name='Offering', - choices=offerings) - offering = propget.get(prop) - print '\nYou have chosen this offering:' - offering.describe() - unit_price = float(offering.fixed_price) - total_price = unit_price * params['quantity'] - print '!!! You are about to purchase %d of these offerings for a total of $%.2f !!!' % (params['quantity'], total_price) - answer = raw_input('Are you sure you want to do this? If so, enter YES: ') - if answer.strip().lower() == 'yes': - offering.purchase(params['quantity']) - else: - print 'Purchase cancelled' diff --git a/backup/src/boto/ec2/cloudwatch/__init__.py b/backup/src/boto/ec2/cloudwatch/__init__.py deleted file mode 100644 index a02baa3..0000000 --- a/backup/src/boto/ec2/cloudwatch/__init__.py +++ /dev/null @@ -1,502 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -""" -This module provides an interface to the Elastic Compute Cloud (EC2) -CloudWatch service from AWS. - -The 5 Minute How-To Guide -------------------------- -First, make sure you have something to monitor. You can either create a -LoadBalancer or enable monitoring on an existing EC2 instance. To enable -monitoring, you can either call the monitor_instance method on the -EC2Connection object or call the monitor method on the Instance object. - -It takes a while for the monitoring data to start accumulating but once -it does, you can do this: - ->>> import boto ->>> c = boto.connect_cloudwatch() ->>> metrics = c.list_metrics() ->>> metrics -[Metric:NetworkIn, - Metric:NetworkOut, - Metric:NetworkOut(InstanceType,m1.small), - Metric:NetworkIn(InstanceId,i-e573e68c), - Metric:CPUUtilization(InstanceId,i-e573e68c), - Metric:DiskWriteBytes(InstanceType,m1.small), - Metric:DiskWriteBytes(ImageId,ami-a1ffb63), - Metric:NetworkOut(ImageId,ami-a1ffb63), - Metric:DiskWriteOps(InstanceType,m1.small), - Metric:DiskReadBytes(InstanceType,m1.small), - Metric:DiskReadOps(ImageId,ami-a1ffb63), - Metric:CPUUtilization(InstanceType,m1.small), - Metric:NetworkIn(ImageId,ami-a1ffb63), - Metric:DiskReadOps(InstanceType,m1.small), - Metric:DiskReadBytes, - Metric:CPUUtilization, - Metric:DiskWriteBytes(InstanceId,i-e573e68c), - Metric:DiskWriteOps(InstanceId,i-e573e68c), - Metric:DiskWriteOps, - Metric:DiskReadOps, - Metric:CPUUtilization(ImageId,ami-a1ffb63), - Metric:DiskReadOps(InstanceId,i-e573e68c), - Metric:NetworkOut(InstanceId,i-e573e68c), - Metric:DiskReadBytes(ImageId,ami-a1ffb63), - Metric:DiskReadBytes(InstanceId,i-e573e68c), - Metric:DiskWriteBytes, - Metric:NetworkIn(InstanceType,m1.small), - Metric:DiskWriteOps(ImageId,ami-a1ffb63)] - -The list_metrics call will return a list of all of the available metrics -that you can query against. Each entry in the list is a Metric object. -As you can see from the list above, some of the metrics are generic metrics -and some have Dimensions associated with them (e.g. InstanceType=m1.small). -The Dimension can be used to refine your query. So, for example, I could -query the metric Metric:CPUUtilization which would create the desired statistic -by aggregating cpu utilization data across all sources of information available -or I could refine that by querying the metric -Metric:CPUUtilization(InstanceId,i-e573e68c) which would use only the data -associated with the instance identified by the instance ID i-e573e68c. - -Because for this example, I'm only monitoring a single instance, the set -of metrics available to me are fairly limited. If I was monitoring many -instances, using many different instance types and AMI's and also several -load balancers, the list of available metrics would grow considerably. - -Once you have the list of available metrics, you can actually -query the CloudWatch system for that metric. Let's choose the CPU utilization -metric for our instance. - ->>> m = metrics[5] ->>> m -Metric:CPUUtilization(InstanceId,i-e573e68c) - -The Metric object has a query method that lets us actually perform -the query against the collected data in CloudWatch. To call that, -we need a start time and end time to control the time span of data -that we are interested in. For this example, let's say we want the -data for the previous hour: - ->>> import datetime ->>> end = datetime.datetime.now() ->>> start = end - datetime.timedelta(hours=1) - -We also need to supply the Statistic that we want reported and -the Units to use for the results. The Statistic can be one of these -values: - -['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount'] - -And Units must be one of the following: - -['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', -'Bytes/Second', 'Bits/Second', 'Count/Second'] - -The query method also takes an optional parameter, period. This -parameter controls the granularity (in seconds) of the data returned. -The smallest period is 60 seconds and the value must be a multiple -of 60 seconds. So, let's ask for the average as a percent: - ->>> datapoints = m.query(start, end, 'Average', 'Percent') ->>> len(datapoints) -60 - -Our period was 60 seconds and our duration was one hour so -we should get 60 data points back and we can see that we did. -Each element in the datapoints list is a DataPoint object -which is a simple subclass of a Python dict object. Each -Datapoint object contains all of the information available -about that particular data point. - ->>> d = datapoints[0] ->>> d -{u'Average': 0.0, - u'SampleCount': 1.0, - u'Timestamp': u'2009-05-21T19:55:00Z', - u'Unit': u'Percent'} - -My server obviously isn't very busy right now! -""" -try: - import simplejson as json -except ImportError: - import json -from boto.connection import AWSQueryConnection -from boto.ec2.cloudwatch.metric import Metric -from boto.ec2.cloudwatch.alarm import MetricAlarm, AlarmHistoryItem -from boto.ec2.cloudwatch.datapoint import Datapoint -from boto.regioninfo import RegionInfo -import boto - -RegionData = { - 'us-east-1' : 'monitoring.us-east-1.amazonaws.com', - 'us-west-1' : 'monitoring.us-west-1.amazonaws.com', - 'eu-west-1' : 'monitoring.eu-west-1.amazonaws.com', - 'ap-southeast-1' : 'monitoring.ap-southeast-1.amazonaws.com'} - -def regions(): - """ - Get all available regions for the CloudWatch service. - - :rtype: list - :return: A list of :class:`boto.RegionInfo` instances - """ - regions = [] - for region_name in RegionData: - region = RegionInfo(name=region_name, - endpoint=RegionData[region_name], - connection_cls=CloudWatchConnection) - regions.append(region) - return regions - -def connect_to_region(region_name, **kw_params): - """ - Given a valid region name, return a - :class:`boto.ec2.cloudwatch.CloudWatchConnection`. - - :param str region_name: The name of the region to connect to. - - :rtype: :class:`boto.ec2.CloudWatchConnection` or ``None`` - :return: A connection to the given region, or None if an invalid region - name is given - """ - for region in regions(): - if region.name == region_name: - return region.connect(**kw_params) - return None - - -class CloudWatchConnection(AWSQueryConnection): - - APIVersion = boto.config.get('Boto', 'cloudwatch_version', '2010-08-01') - DefaultRegionName = boto.config.get('Boto', 'cloudwatch_region_name', 'us-east-1') - DefaultRegionEndpoint = boto.config.get('Boto', 'cloudwatch_region_endpoint', - 'monitoring.amazonaws.com') - - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - """ - Init method to create a new connection to EC2 Monitoring Service. - - B{Note:} The host argument is overridden by the host specified in the - boto configuration file. - """ - if not region: - region = RegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint) - self.region = region - - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['ec2'] - - def build_list_params(self, params, items, label): - if isinstance(items, str): - items = [items] - for i in range(1, len(items)+1): - params[label % i] = items[i-1] - - def get_metric_statistics(self, period, start_time, end_time, metric_name, - namespace, statistics, dimensions=None, unit=None): - """ - Get time-series data for one or more statistics of a given metric. - - :type metric_name: string - :param metric_name: CPUUtilization|NetworkIO-in|NetworkIO-out|DiskIO-ALL-read| - DiskIO-ALL-write|DiskIO-ALL-read-bytes|DiskIO-ALL-write-bytes - - :rtype: list - """ - params = {'Period' : period, - 'MetricName' : metric_name, - 'Namespace' : namespace, - 'StartTime' : start_time.isoformat(), - 'EndTime' : end_time.isoformat()} - self.build_list_params(params, statistics, 'Statistics.member.%d') - if dimensions: - i = 1 - for name in dimensions: - params['Dimensions.member.%d.Name' % i] = name - params['Dimensions.member.%d.Value' % i] = dimensions[name] - i += 1 - return self.get_list('GetMetricStatistics', params, [('member', Datapoint)]) - - def list_metrics(self, next_token=None): - """ - Returns a list of the valid metrics for which there is recorded data available. - - :type next_token: string - :param next_token: A maximum of 500 metrics will be returned at one time. - If more results are available, the ResultSet returned - will contain a non-Null next_token attribute. Passing - that token as a parameter to list_metrics will retrieve - the next page of metrics. - """ - params = {} - if next_token: - params['NextToken'] = next_token - return self.get_list('ListMetrics', params, [('member', Metric)]) - - def describe_alarms(self, action_prefix=None, alarm_name_prefix=None, alarm_names=None, - max_records=None, state_value=None, next_token=None): - """ - Retrieves alarms with the specified names. If no name is specified, all - alarms for the user are returned. Alarms can be retrieved by using only - a prefix for the alarm name, the alarm state, or a prefix for any - action. - - :type action_prefix: string - :param action_name: The action name prefix. - - :type alarm_name_prefix: string - :param alarm_name_prefix: The alarm name prefix. AlarmNames cannot be specified - if this parameter is specified. - - :type alarm_names: list - :param alarm_names: A list of alarm names to retrieve information for. - - :type max_records: int - :param max_records: The maximum number of alarm descriptions to retrieve. - - :type state_value: string - :param state_value: The state value to be used in matching alarms. - - :type next_token: string - :param next_token: The token returned by a previous call to indicate that there is more data. - - :rtype list - """ - params = {} - if action_prefix: - params['ActionPrefix'] = action_prefix - if alarm_name_prefix: - params['AlarmNamePrefix'] = alarm_name_prefix - elif alarm_names: - self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') - if max_records: - params['MaxRecords'] = max_records - if next_token: - params['NextToken'] = next_token - if state_value: - params['StateValue'] = state_value - return self.get_list('DescribeAlarms', params, [('member', MetricAlarm)]) - - def describe_alarm_history(self, alarm_name=None, start_date=None, end_date=None, - max_records=None, history_item_type=None, next_token=None): - """ - Retrieves history for the specified alarm. Filter alarms by date range - or item type. If an alarm name is not specified, Amazon CloudWatch - returns histories for all of the owner's alarms. - - Amazon CloudWatch retains the history of deleted alarms for a period of - six weeks. If an alarm has been deleted, its history can still be - queried. - - :type alarm_name: string - :param alarm_name: The name of the alarm. - - :type start_date: datetime - :param start_date: The starting date to retrieve alarm history. - - :type end_date: datetime - :param end_date: The starting date to retrieve alarm history. - - :type history_item_type: string - :param history_item_type: The type of alarm histories to retreive (ConfigurationUpdate | StateUpdate | Action) - - :type max_records: int - :param max_records: The maximum number of alarm descriptions to retrieve. - - :type next_token: string - :param next_token: The token returned by a previous call to indicate that there is more data. - - :rtype list - """ - params = {} - if alarm_name: - params['AlarmName'] = alarm_name - if start_date: - params['StartDate'] = start_date.isoformat() - if end_date: - params['EndDate'] = end_date.isoformat() - if history_item_type: - params['HistoryItemType'] = history_item_type - if max_records: - params['MaxRecords'] = max_records - if next_token: - params['NextToken'] = next_token - return self.get_list('DescribeAlarmHistory', params, [('member', AlarmHistoryItem)]) - - def describe_alarms_for_metric(self, metric_name, namespace, period=None, statistic=None, dimensions=None, unit=None): - """ - Retrieves all alarms for a single metric. Specify a statistic, period, - or unit to filter the set of alarms further. - - :type metric_name: string - :param metric_name: The name of the metric - - :type namespace: string - :param namespace: The namespace of the metric. - - :type period: int - :param period: The period in seconds over which the statistic is applied. - - :type statistic: string - :param statistic: The statistic for the metric. - - :type dimensions: list - - :type unit: string - - :rtype list - """ - params = { - 'MetricName' : metric_name, - 'Namespace' : namespace, - } - if period: - params['Period'] = period - if statistic: - params['Statistic'] = statistic - if dimensions: - self.build_list_params(params, dimensions, 'Dimensions.member.%s') - if unit: - params['Unit'] = unit - return self.get_list('DescribeAlarmsForMetric', params, [('member', MetricAlarm)]) - - def put_metric_alarm(self, alarm): - """ - Creates or updates an alarm and associates it with the specified Amazon - CloudWatch metric. Optionally, this operation can associate one or more - Amazon Simple Notification Service resources with the alarm. - - When this operation creates an alarm, the alarm state is immediately - set to INSUFFICIENT_DATA. The alarm is evaluated and its StateValue is - set appropriately. Any actions associated with the StateValue is then - executed. - - When updating an existing alarm, its StateValue is left unchanged. - - :type alarm: boto.ec2.cloudwatch.alarm.MetricAlarm - :param alarm: MetricAlarm object. - """ - params = { - 'AlarmName' : alarm.name, - 'MetricName' : alarm.metric, - 'Namespace' : alarm.namespace, - 'Statistic' : alarm.statistic, - 'ComparisonOperator' : MetricAlarm._cmp_map[alarm.comparison], - 'Threshold' : alarm.threshold, - 'EvaluationPeriods' : alarm.evaluation_periods, - 'Period' : alarm.period, - } - if alarm.actions_enabled is not None: - params['ActionsEnabled'] = alarm.actions_enabled - if alarm.alarm_actions: - self.build_list_params(params, alarm.alarm_actions, 'AlarmActions.member.%s') - if alarm.description: - params['AlarmDescription'] = alarm.description - if alarm.dimensions: - self.build_list_params(params, alarm.dimensions, 'Dimensions.member.%s') - if alarm.insufficient_data_actions: - self.build_list_params(params, alarm.insufficient_data_actions, 'InsufficientDataActions.member.%s') - if alarm.ok_actions: - self.build_list_params(params, alarm.ok_actions, 'OKActions.member.%s') - if alarm.unit: - params['Unit'] = alarm.unit - alarm.connection = self - return self.get_status('PutMetricAlarm', params) - create_alarm = put_metric_alarm - update_alarm = put_metric_alarm - - def delete_alarms(self, alarms): - """ - Deletes all specified alarms. In the event of an error, no alarms are deleted. - - :type alarms: list - :param alarms: List of alarm names. - """ - params = {} - self.build_list_params(params, alarms, 'AlarmNames.member.%s') - return self.get_status('DeleteAlarms', params) - - def set_alarm_state(self, alarm_name, state_reason, state_value, state_reason_data=None): - """ - Temporarily sets the state of an alarm. When the updated StateValue - differs from the previous value, the action configured for the - appropriate state is invoked. This is not a permanent change. The next - periodic alarm check (in about a minute) will set the alarm to its - actual state. - - :type alarm_name: string - :param alarm_name: Descriptive name for alarm. - - :type state_reason: string - :param state_reason: Human readable reason. - - :type state_value: string - :param state_value: OK | ALARM | INSUFFICIENT_DATA - - :type state_reason_data: string - :param state_reason_data: Reason string (will be jsonified). - """ - params = { - 'AlarmName' : alarm_name, - 'StateReason' : state_reason, - 'StateValue' : state_value, - } - if state_reason_data: - params['StateReasonData'] = json.dumps(state_reason_data) - - return self.get_status('SetAlarmState', params) - - def enable_alarm_actions(self, alarm_names): - """ - Enables actions for the specified alarms. - - :type alarms: list - :param alarms: List of alarm names. - """ - params = {} - self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') - return self.get_status('EnableAlarmActions', params) - - def disable_alarm_actions(self, alarm_names): - """ - Disables actions for the specified alarms. - - :type alarms: list - :param alarms: List of alarm names. - """ - params = {} - self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') - return self.get_status('DisableAlarmActions', params) - diff --git a/backup/src/boto/ec2/cloudwatch/alarm.py b/backup/src/boto/ec2/cloudwatch/alarm.py deleted file mode 100644 index a43af22..0000000 --- a/backup/src/boto/ec2/cloudwatch/alarm.py +++ /dev/null @@ -1,189 +0,0 @@ -# Copyright (c) 2010 Reza Lotun http://reza.lotun.name -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from datetime import datetime -from boto.resultset import ResultSet -from boto.ec2.cloudwatch.listelement import ListElement -import json - - -class MetricAlarm(object): - - OK = 'OK' - ALARM = 'ALARM' - INSUFFICIENT_DATA = 'INSUFFICIENT_DATA' - - _cmp_map = { - '>=' : 'GreaterThanOrEqualToThreshold', - '>' : 'GreaterThanThreshold', - '<' : 'LessThanThreshold', - '<=' : 'LessThanOrEqualToThreshold', - } - _rev_cmp_map = dict((v, k) for (k, v) in _cmp_map.iteritems()) - - def __init__(self, connection=None, name=None, metric=None, - namespace=None, statistic=None, comparison=None, threshold=None, - period=None, evaluation_periods=None): - """ - Creates a new Alarm. - - :type name: str - :param name: Name of alarm. - - :type metric: str - :param metric: Name of alarm's associated metric. - - :type namespace: str - :param namespace: The namespace for the alarm's metric. - - :type statistic: str - :param statistic: The statistic to apply to the alarm's associated metric. Can - be one of 'SampleCount', 'Average', 'Sum', 'Minimum', 'Maximum' - - :type comparison: str - :param comparison: Comparison used to compare statistic with threshold. Can be - one of '>=', '>', '<', '<=' - - :type threshold: float - :param threshold: The value against which the specified statistic is compared. - - :type period: int - :param period: The period in seconds over which teh specified statistic is applied. - - :type evaluation_periods: int - :param evaluation_period: The number of periods over which data is compared to - the specified threshold - """ - self.name = name - self.connection = connection - self.metric = metric - self.namespace = namespace - self.statistic = statistic - self.threshold = float(threshold) if threshold is not None else None - self.comparison = self._cmp_map.get(comparison) - self.period = int(period) if period is not None else None - self.evaluation_periods = int(evaluation_periods) if evaluation_periods is not None else None - self.actions_enabled = None - self.alarm_arn = None - self.last_updated = None - self.description = '' - self.dimensions = [] - self.insufficient_data_actions = [] - self.ok_actions = [] - self.state_reason = None - self.state_value = None - self.unit = None - alarm_action = [] - self.alarm_actions = ListElement(alarm_action) - - def __repr__(self): - return 'MetricAlarm:%s[%s(%s) %s %s]' % (self.name, self.metric, self.statistic, self.comparison, self.threshold) - - def startElement(self, name, attrs, connection): - if name == 'AlarmActions': - return self.alarm_actions - else: - pass - - def endElement(self, name, value, connection): - if name == 'ActionsEnabled': - self.actions_enabled = value - elif name == 'AlarmArn': - self.alarm_arn = value - elif name == 'AlarmConfigurationUpdatedTimestamp': - self.last_updated = value - elif name == 'AlarmDescription': - self.description = value - elif name == 'AlarmName': - self.name = value - elif name == 'ComparisonOperator': - setattr(self, 'comparison', self._rev_cmp_map[value]) - elif name == 'EvaluationPeriods': - self.evaluation_periods = int(value) - elif name == 'MetricName': - self.metric = value - elif name == 'Namespace': - self.namespace = value - elif name == 'Period': - self.period = int(value) - elif name == 'StateReason': - self.state_reason = value - elif name == 'StateValue': - self.state_value = value - elif name == 'Statistic': - self.statistic = value - elif name == 'Threshold': - self.threshold = float(value) - elif name == 'Unit': - self.unit = value - else: - setattr(self, name, value) - - def set_state(self, value, reason, data=None): - """ Temporarily sets the state of an alarm. - - :type value: str - :param value: OK | ALARM | INSUFFICIENT_DATA - - :type reason: str - :param reason: Reason alarm set (human readable). - - :type data: str - :param data: Reason data (will be jsonified). - """ - return self.connection.set_alarm_state(self.name, reason, value, data) - - def update(self): - return self.connection.update_alarm(self) - - def enable_actions(self): - return self.connection.enable_alarm_actions([self.name]) - - def disable_actions(self): - return self.connection.disable_alarm_actions([self.name]) - - def describe_history(self, start_date=None, end_date=None, max_records=None, history_item_type=None, next_token=None): - return self.connection.describe_alarm_history(self.name, start_date, end_date, - max_records, history_item_type, next_token) - - -class AlarmHistoryItem(object): - def __init__(self, connection=None): - self.connection = connection - - def __repr__(self): - return 'AlarmHistory:%s[%s at %s]' % (self.name, self.summary, self.timestamp) - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'AlarmName': - self.name = value - elif name == 'HistoryData': - self.data = json.loads(value) - elif name == 'HistoryItemType': - self.tem_type = value - elif name == 'HistorySummary': - self.summary = value - elif name == 'Timestamp': - self.timestamp = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ') diff --git a/backup/src/boto/ec2/cloudwatch/datapoint.py b/backup/src/boto/ec2/cloudwatch/datapoint.py deleted file mode 100644 index d4350ce..0000000 --- a/backup/src/boto/ec2/cloudwatch/datapoint.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -from datetime import datetime - -class Datapoint(dict): - - def __init__(self, connection=None): - dict.__init__(self) - self.connection = connection - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name in ['Average', 'Maximum', 'Minimum', 'Sum', 'SampleCount']: - self[name] = float(value) - elif name == 'Timestamp': - self[name] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ') - elif name != 'member': - self[name] = value - diff --git a/backup/src/boto/ec2/cloudwatch/listelement.py b/backup/src/boto/ec2/cloudwatch/listelement.py deleted file mode 100644 index 5be4599..0000000 --- a/backup/src/boto/ec2/cloudwatch/listelement.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class ListElement(list): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'member': - self.append(value) - - diff --git a/backup/src/boto/ec2/cloudwatch/metric.py b/backup/src/boto/ec2/cloudwatch/metric.py deleted file mode 100644 index cd8c4bc..0000000 --- a/backup/src/boto/ec2/cloudwatch/metric.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -class Dimensions(dict): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'Name': - self._name = value - elif name == 'Value': - self[self._name] = value - elif name != 'Dimensions' and name != 'member': - self[name] = value - -class Metric(object): - - Statistics = ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount'] - Units = ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', - 'Bytes/Second', 'Bits/Second', 'Count/Second'] - - def __init__(self, connection=None): - self.connection = connection - self.name = None - self.namespace = None - self.dimensions = None - - def __repr__(self): - s = 'Metric:%s' % self.name - if self.dimensions: - for name,value in self.dimensions.items(): - s += '(%s,%s)' % (name, value) - return s - - def startElement(self, name, attrs, connection): - if name == 'Dimensions': - self.dimensions = Dimensions() - return self.dimensions - - def endElement(self, name, value, connection): - if name == 'MetricName': - self.name = value - elif name == 'Namespace': - self.namespace = value - else: - setattr(self, name, value) - - def query(self, start_time, end_time, statistic, unit=None, period=60): - return self.connection.get_metric_statistics(period, start_time, end_time, - self.name, self.namespace, [statistic], - self.dimensions, unit) - - def describe_alarms(self, period=None, statistic=None, dimensions=None, unit=None): - return self.connection.describe_alarms_for_metric(self.name, - self.namespace, - period, - statistic, - dimensions, - unit) - diff --git a/backup/src/boto/ec2/connection.py b/backup/src/boto/ec2/connection.py deleted file mode 100644 index 6ebbef7..0000000 --- a/backup/src/boto/ec2/connection.py +++ /dev/null @@ -1,2319 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a connection to the EC2 service. -""" - -import base64 -import warnings -from datetime import datetime -from datetime import timedelta -import boto -from boto.connection import AWSQueryConnection -from boto.resultset import ResultSet -from boto.ec2.image import Image, ImageAttribute -from boto.ec2.instance import Reservation, Instance, ConsoleOutput, InstanceAttribute -from boto.ec2.keypair import KeyPair -from boto.ec2.address import Address -from boto.ec2.volume import Volume -from boto.ec2.snapshot import Snapshot -from boto.ec2.snapshot import SnapshotAttribute -from boto.ec2.zone import Zone -from boto.ec2.securitygroup import SecurityGroup -from boto.ec2.regioninfo import RegionInfo -from boto.ec2.instanceinfo import InstanceInfo -from boto.ec2.reservedinstance import ReservedInstancesOffering, ReservedInstance -from boto.ec2.spotinstancerequest import SpotInstanceRequest -from boto.ec2.spotpricehistory import SpotPriceHistory -from boto.ec2.spotdatafeedsubscription import SpotDatafeedSubscription -from boto.ec2.bundleinstance import BundleInstanceTask -from boto.ec2.placementgroup import PlacementGroup -from boto.ec2.tag import Tag -from boto.exception import EC2ResponseError - -#boto.set_stream_logger('ec2') - -class EC2Connection(AWSQueryConnection): - - APIVersion = boto.config.get('Boto', 'ec2_version', '2010-08-31') - DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1') - DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint', - 'ec2.amazonaws.com') - ResponseError = EC2ResponseError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, host=None, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - """ - Init method to create a new connection to EC2. - - B{Note:} The host argument is overridden by the host specified in the - boto configuration file. - """ - if not region: - region = RegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['ec2'] - - def get_params(self): - """ - Returns a dictionary containing the value of of all of the keyword - arguments passed when constructing this connection. - """ - param_names = ['aws_access_key_id', 'aws_secret_access_key', 'is_secure', - 'port', 'proxy', 'proxy_port', 'proxy_user', 'proxy_pass', - 'debug', 'https_connection_factory'] - params = {} - for name in param_names: - params[name] = getattr(self, name) - return params - - def build_filter_params(self, params, filters): - i = 1 - for name in filters: - aws_name = name.replace('_', '-') - params['Filter.%d.Name' % i] = aws_name - value = filters[name] - if not isinstance(value, list): - value = [value] - j = 1 - for v in value: - params['Filter.%d.Value.%d' % (i,j)] = v - j += 1 - i += 1 - - # Image methods - - def get_all_images(self, image_ids=None, owners=None, - executable_by=None, filters=None): - """ - Retrieve all the EC2 images available on your account. - - :type image_ids: list - :param image_ids: A list of strings with the image IDs wanted - - :type owners: list - :param owners: A list of owner IDs - - :type executable_by: list - :param executable_by: Returns AMIs for which the specified - user ID has explicit launch permissions - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.image.Image` - """ - params = {} - if image_ids: - self.build_list_params(params, image_ids, 'ImageId') - if owners: - self.build_list_params(params, owners, 'Owner') - if executable_by: - self.build_list_params(params, executable_by, 'ExecutableBy') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') - - def get_all_kernels(self, kernel_ids=None, owners=None): - """ - Retrieve all the EC2 kernels available on your account. - Constructs a filter to allow the processing to happen server side. - - :type kernel_ids: list - :param kernel_ids: A list of strings with the image IDs wanted - - :type owners: list - :param owners: A list of owner IDs - - :rtype: list - :return: A list of :class:`boto.ec2.image.Image` - """ - params = {} - if kernel_ids: - self.build_list_params(params, kernel_ids, 'ImageId') - if owners: - self.build_list_params(params, owners, 'Owner') - filter = {'image-type' : 'kernel'} - self.build_filter_params(params, filter) - return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') - - def get_all_ramdisks(self, ramdisk_ids=None, owners=None): - """ - Retrieve all the EC2 ramdisks available on your account. - Constructs a filter to allow the processing to happen server side. - - :type ramdisk_ids: list - :param ramdisk_ids: A list of strings with the image IDs wanted - - :type owners: list - :param owners: A list of owner IDs - - :rtype: list - :return: A list of :class:`boto.ec2.image.Image` - """ - params = {} - if ramdisk_ids: - self.build_list_params(params, ramdisk_ids, 'ImageId') - if owners: - self.build_list_params(params, owners, 'Owner') - filter = {'image-type' : 'ramdisk'} - self.build_filter_params(params, filter) - return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') - - def get_image(self, image_id): - """ - Shortcut method to retrieve a specific image (AMI). - - :type image_id: string - :param image_id: the ID of the Image to retrieve - - :rtype: :class:`boto.ec2.image.Image` - :return: The EC2 Image specified or None if the image is not found - """ - try: - return self.get_all_images(image_ids=[image_id])[0] - except IndexError: # None of those images available - return None - - def register_image(self, name=None, description=None, image_location=None, - architecture=None, kernel_id=None, ramdisk_id=None, - root_device_name=None, block_device_map=None): - """ - Register an image. - - :type name: string - :param name: The name of the AMI. Valid only for EBS-based images. - - :type description: string - :param description: The description of the AMI. - - :type image_location: string - :param image_location: Full path to your AMI manifest in Amazon S3 storage. - Only used for S3-based AMI's. - - :type architecture: string - :param architecture: The architecture of the AMI. Valid choices are: - i386 | x86_64 - - :type kernel_id: string - :param kernel_id: The ID of the kernel with which to launch the instances - - :type root_device_name: string - :param root_device_name: The root device name (e.g. /dev/sdh) - - :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` - :param block_device_map: A BlockDeviceMapping data structure - describing the EBS volumes associated - with the Image. - - :rtype: string - :return: The new image id - """ - params = {} - if name: - params['Name'] = name - if description: - params['Description'] = description - if architecture: - params['Architecture'] = architecture - if kernel_id: - params['KernelId'] = kernel_id - if ramdisk_id: - params['RamdiskId'] = ramdisk_id - if image_location: - params['ImageLocation'] = image_location - if root_device_name: - params['RootDeviceName'] = root_device_name - if block_device_map: - block_device_map.build_list_params(params) - rs = self.get_object('RegisterImage', params, ResultSet, verb='POST') - image_id = getattr(rs, 'imageId', None) - return image_id - - def deregister_image(self, image_id): - """ - Unregister an AMI. - - :type image_id: string - :param image_id: the ID of the Image to unregister - - :rtype: bool - :return: True if successful - """ - return self.get_status('DeregisterImage', {'ImageId':image_id}, verb='POST') - - def create_image(self, instance_id, name, description=None, no_reboot=False): - """ - Will create an AMI from the instance in the running or stopped - state. - - :type instance_id: string - :param instance_id: the ID of the instance to image. - - :type name: string - :param name: The name of the new image - - :type description: string - :param description: An optional human-readable string describing - the contents and purpose of the AMI. - - :type no_reboot: bool - :param no_reboot: An optional flag indicating that the bundling process - should not attempt to shutdown the instance before - bundling. If this flag is True, the responsibility - of maintaining file system integrity is left to the - owner of the instance. - - :rtype: string - :return: The new image id - """ - params = {'InstanceId' : instance_id, - 'Name' : name} - if description: - params['Description'] = description - if no_reboot: - params['NoReboot'] = 'true' - img = self.get_object('CreateImage', params, Image, verb='POST') - return img.id - - # ImageAttribute methods - - def get_image_attribute(self, image_id, attribute='launchPermission'): - """ - Gets an attribute from an image. - - :type image_id: string - :param image_id: The Amazon image id for which you want info about - - :type attribute: string - :param attribute: The attribute you need information about. - Valid choices are: - * launchPermission - * productCodes - * blockDeviceMapping - - :rtype: :class:`boto.ec2.image.ImageAttribute` - :return: An ImageAttribute object representing the value of the - attribute requested - """ - params = {'ImageId' : image_id, - 'Attribute' : attribute} - return self.get_object('DescribeImageAttribute', params, ImageAttribute, verb='POST') - - def modify_image_attribute(self, image_id, attribute='launchPermission', - operation='add', user_ids=None, groups=None, - product_codes=None): - """ - Changes an attribute of an image. - - :type image_id: string - :param image_id: The image id you wish to change - - :type attribute: string - :param attribute: The attribute you wish to change - - :type operation: string - :param operation: Either add or remove (this is required for changing - launchPermissions) - - :type user_ids: list - :param user_ids: The Amazon IDs of users to add/remove attributes - - :type groups: list - :param groups: The groups to add/remove attributes - - :type product_codes: list - :param product_codes: Amazon DevPay product code. Currently only one - product code can be associated with an AMI. Once - set, the product code cannot be changed or reset. - """ - params = {'ImageId' : image_id, - 'Attribute' : attribute, - 'OperationType' : operation} - if user_ids: - self.build_list_params(params, user_ids, 'UserId') - if groups: - self.build_list_params(params, groups, 'UserGroup') - if product_codes: - self.build_list_params(params, product_codes, 'ProductCode') - return self.get_status('ModifyImageAttribute', params, verb='POST') - - def reset_image_attribute(self, image_id, attribute='launchPermission'): - """ - Resets an attribute of an AMI to its default value. - - :type image_id: string - :param image_id: ID of the AMI for which an attribute will be described - - :type attribute: string - :param attribute: The attribute to reset - - :rtype: bool - :return: Whether the operation succeeded or not - """ - params = {'ImageId' : image_id, - 'Attribute' : attribute} - return self.get_status('ResetImageAttribute', params, verb='POST') - - # Instance methods - - def get_all_instances(self, instance_ids=None, filters=None): - """ - Retrieve all the instances associated with your account. - - :type instance_ids: list - :param instance_ids: A list of strings of instance IDs - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.instance.Reservation` - """ - params = {} - if instance_ids: - self.build_list_params(params, instance_ids, 'InstanceId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeInstances', params, - [('item', Reservation)], verb='POST') - - def run_instances(self, image_id, min_count=1, max_count=1, - key_name=None, security_groups=None, - user_data=None, addressing_type=None, - instance_type='m1.small', placement=None, - kernel_id=None, ramdisk_id=None, - monitoring_enabled=False, subnet_id=None, - block_device_map=None, - disable_api_termination=False, - instance_initiated_shutdown_behavior=None, - private_ip_address=None, - placement_group=None, client_token=None): - """ - Runs an image on EC2. - - :type image_id: string - :param image_id: The ID of the image to run - - :type min_count: int - :param min_count: The minimum number of instances to launch - - :type max_count: int - :param max_count: The maximum number of instances to launch - - :type key_name: string - :param key_name: The name of the key pair with which to launch instances - - :type security_groups: list of strings - :param security_groups: The names of the security groups with which to - associate instances - - :type user_data: string - :param user_data: The user data passed to the launched instances - - :type instance_type: string - :param instance_type: The type of instance to run: - - * m1.small - * m1.large - * m1.xlarge - * c1.medium - * c1.xlarge - * m2.xlarge - * m2.2xlarge - * m2.4xlarge - * cc1.4xlarge - * t1.micro - - :type placement: string - :param placement: The availability zone in which to launch the instances - - :type kernel_id: string - :param kernel_id: The ID of the kernel with which to launch the - instances - - :type ramdisk_id: string - :param ramdisk_id: The ID of the RAM disk with which to launch the - instances - - :type monitoring_enabled: bool - :param monitoring_enabled: Enable CloudWatch monitoring on the instance. - - :type subnet_id: string - :param subnet_id: The subnet ID within which to launch the instances - for VPC. - - :type private_ip_address: string - :param private_ip_address: If you're using VPC, you can optionally use - this parameter to assign the instance a - specific available IP address from the - subnet (e.g., 10.0.0.25). - - :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` - :param block_device_map: A BlockDeviceMapping data structure - describing the EBS volumes associated - with the Image. - - :type disable_api_termination: bool - :param disable_api_termination: If True, the instances will be locked - and will not be able to be terminated - via the API. - - :type instance_initiated_shutdown_behavior: string - :param instance_initiated_shutdown_behavior: Specifies whether the - instance's EBS volumes are - stopped (i.e. detached) or - terminated (i.e. deleted) - when the instance is - shutdown by the - owner. Valid values are: - - * stop - * terminate - - :type placement_group: string - :param placement_group: If specified, this is the name of the placement - group in which the instance(s) will be launched. - - :type client_token: string - :param client_token: Unique, case-sensitive identifier you provide - to ensure idempotency of the request. - Maximum 64 ASCII characters - - :rtype: Reservation - :return: The :class:`boto.ec2.instance.Reservation` associated with - the request for machines - """ - params = {'ImageId':image_id, - 'MinCount':min_count, - 'MaxCount': max_count} - if key_name: - params['KeyName'] = key_name - if security_groups: - l = [] - for group in security_groups: - if isinstance(group, SecurityGroup): - l.append(group.name) - else: - l.append(group) - self.build_list_params(params, l, 'SecurityGroup') - if user_data: - params['UserData'] = base64.b64encode(user_data) - if addressing_type: - params['AddressingType'] = addressing_type - if instance_type: - params['InstanceType'] = instance_type - if placement: - params['Placement.AvailabilityZone'] = placement - if placement_group: - params['Placement.GroupName'] = placement_group - if kernel_id: - params['KernelId'] = kernel_id - if ramdisk_id: - params['RamdiskId'] = ramdisk_id - if monitoring_enabled: - params['Monitoring.Enabled'] = 'true' - if subnet_id: - params['SubnetId'] = subnet_id - if private_ip_address: - params['PrivateIpAddress'] = private_ip_address - if block_device_map: - block_device_map.build_list_params(params) - if disable_api_termination: - params['DisableApiTermination'] = 'true' - if instance_initiated_shutdown_behavior: - val = instance_initiated_shutdown_behavior - params['InstanceInitiatedShutdownBehavior'] = val - if client_token: - params['ClientToken'] = client_token - return self.get_object('RunInstances', params, Reservation, verb='POST') - - def terminate_instances(self, instance_ids=None): - """ - Terminate the instances specified - - :type instance_ids: list - :param instance_ids: A list of strings of the Instance IDs to terminate - - :rtype: list - :return: A list of the instances terminated - """ - params = {} - if instance_ids: - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_list('TerminateInstances', params, [('item', Instance)], verb='POST') - - def stop_instances(self, instance_ids=None, force=False): - """ - Stop the instances specified - - :type instance_ids: list - :param instance_ids: A list of strings of the Instance IDs to stop - - :type force: bool - :param force: Forces the instance to stop - - :rtype: list - :return: A list of the instances stopped - """ - params = {} - if force: - params['Force'] = 'true' - if instance_ids: - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_list('StopInstances', params, [('item', Instance)], verb='POST') - - def start_instances(self, instance_ids=None): - """ - Start the instances specified - - :type instance_ids: list - :param instance_ids: A list of strings of the Instance IDs to start - - :rtype: list - :return: A list of the instances started - """ - params = {} - if instance_ids: - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_list('StartInstances', params, [('item', Instance)], verb='POST') - - def get_console_output(self, instance_id): - """ - Retrieves the console output for the specified instance. - - :type instance_id: string - :param instance_id: The instance ID of a running instance on the cloud. - - :rtype: :class:`boto.ec2.instance.ConsoleOutput` - :return: The console output as a ConsoleOutput object - """ - params = {} - self.build_list_params(params, [instance_id], 'InstanceId') - return self.get_object('GetConsoleOutput', params, ConsoleOutput, verb='POST') - - def reboot_instances(self, instance_ids=None): - """ - Reboot the specified instances. - - :type instance_ids: list - :param instance_ids: The instances to terminate and reboot - """ - params = {} - if instance_ids: - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_status('RebootInstances', params) - - def confirm_product_instance(self, product_code, instance_id): - params = {'ProductCode' : product_code, - 'InstanceId' : instance_id} - rs = self.get_object('ConfirmProductInstance', params, ResultSet, verb='POST') - return (rs.status, rs.ownerId) - - # InstanceAttribute methods - - def get_instance_attribute(self, instance_id, attribute): - """ - Gets an attribute from an instance. - - :type instance_id: string - :param instance_id: The Amazon id of the instance - - :type attribute: string - :param attribute: The attribute you need information about - Valid choices are: - - * instanceType|kernel|ramdisk|userData| - * disableApiTermination| - * instanceInitiatedShutdownBehavior| - * rootDeviceName|blockDeviceMapping - - :rtype: :class:`boto.ec2.image.InstanceAttribute` - :return: An InstanceAttribute object representing the value of the - attribute requested - """ - params = {'InstanceId' : instance_id} - if attribute: - params['Attribute'] = attribute - return self.get_object('DescribeInstanceAttribute', params, - InstanceAttribute, verb='POST') - - def modify_instance_attribute(self, instance_id, attribute, value): - """ - Changes an attribute of an instance - - :type instance_id: string - :param instance_id: The instance id you wish to change - - :type attribute: string - :param attribute: The attribute you wish to change. - - * AttributeName - Expected value (default) - * instanceType - A valid instance type (m1.small) - * kernel - Kernel ID (None) - * ramdisk - Ramdisk ID (None) - * userData - Base64 encoded String (None) - * disableApiTermination - Boolean (true) - * instanceInitiatedShutdownBehavior - stop|terminate - * rootDeviceName - device name (None) - - :type value: string - :param value: The new value for the attribute - - :rtype: bool - :return: Whether the operation succeeded or not - """ - # Allow a bool to be passed in for value of disableApiTermination - if attribute == 'disableApiTermination': - if isinstance(value, bool): - if value: - value = 'true' - else: - value = 'false' - params = {'InstanceId' : instance_id, - 'Attribute' : attribute, - 'Value' : value} - return self.get_status('ModifyInstanceAttribute', params, verb='POST') - - def reset_instance_attribute(self, instance_id, attribute): - """ - Resets an attribute of an instance to its default value. - - :type instance_id: string - :param instance_id: ID of the instance - - :type attribute: string - :param attribute: The attribute to reset. Valid values are: - kernel|ramdisk - - :rtype: bool - :return: Whether the operation succeeded or not - """ - params = {'InstanceId' : instance_id, - 'Attribute' : attribute} - return self.get_status('ResetInstanceAttribute', params, verb='POST') - - # Spot Instances - - def get_all_spot_instance_requests(self, request_ids=None, - filters=None): - """ - Retrieve all the spot instances requests associated with your account. - - :type request_ids: list - :param request_ids: A list of strings of spot instance request IDs - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of - :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` - """ - params = {} - if request_ids: - self.build_list_params(params, request_ids, 'SpotInstanceRequestId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeSpotInstanceRequests', params, - [('item', SpotInstanceRequest)], verb='POST') - - def get_spot_price_history(self, start_time=None, end_time=None, - instance_type=None, product_description=None): - """ - Retrieve the recent history of spot instances pricing. - - :type start_time: str - :param start_time: An indication of how far back to provide price - changes for. An ISO8601 DateTime string. - - :type end_time: str - :param end_time: An indication of how far forward to provide price - changes for. An ISO8601 DateTime string. - - :type instance_type: str - :param instance_type: Filter responses to a particular instance type. - - :type product_description: str - :param product_descripton: Filter responses to a particular platform. - Valid values are currently: Linux - - :rtype: list - :return: A list tuples containing price and timestamp. - """ - params = {} - if start_time: - params['StartTime'] = start_time - if end_time: - params['EndTime'] = end_time - if instance_type: - params['InstanceType'] = instance_type - if product_description: - params['ProductDescription'] = product_description - return self.get_list('DescribeSpotPriceHistory', params, - [('item', SpotPriceHistory)], verb='POST') - - def request_spot_instances(self, price, image_id, count=1, type='one-time', - valid_from=None, valid_until=None, - launch_group=None, availability_zone_group=None, - key_name=None, security_groups=None, - user_data=None, addressing_type=None, - instance_type='m1.small', placement=None, - kernel_id=None, ramdisk_id=None, - monitoring_enabled=False, subnet_id=None, - block_device_map=None): - """ - Request instances on the spot market at a particular price. - - :type price: str - :param price: The maximum price of your bid - - :type image_id: string - :param image_id: The ID of the image to run - - :type count: int - :param count: The of instances to requested - - :type type: str - :param type: Type of request. Can be 'one-time' or 'persistent'. - Default is one-time. - - :type valid_from: str - :param valid_from: Start date of the request. An ISO8601 time string. - - :type valid_until: str - :param valid_until: End date of the request. An ISO8601 time string. - - :type launch_group: str - :param launch_group: If supplied, all requests will be fulfilled - as a group. - - :type availability_zone_group: str - :param availability_zone_group: If supplied, all requests will be - fulfilled within a single - availability zone. - - :type key_name: string - :param key_name: The name of the key pair with which to launch instances - - :type security_groups: list of strings - :param security_groups: The names of the security groups with which to - associate instances - - :type user_data: string - :param user_data: The user data passed to the launched instances - - :type instance_type: string - :param instance_type: The type of instance to run: - - * m1.small - * m1.large - * m1.xlarge - * c1.medium - * c1.xlarge - * m2.xlarge - * m2.2xlarge - * m2.4xlarge - * cc1.4xlarge - * t1.micro - - :type placement: string - :param placement: The availability zone in which to launch the instances - - :type kernel_id: string - :param kernel_id: The ID of the kernel with which to launch the - instances - - :type ramdisk_id: string - :param ramdisk_id: The ID of the RAM disk with which to launch the - instances - - :type monitoring_enabled: bool - :param monitoring_enabled: Enable CloudWatch monitoring on the instance. - - :type subnet_id: string - :param subnet_id: The subnet ID within which to launch the instances - for VPC. - - :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` - :param block_device_map: A BlockDeviceMapping data structure - describing the EBS volumes associated - with the Image. - - :rtype: Reservation - :return: The :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` - associated with the request for machines - """ - params = {'LaunchSpecification.ImageId':image_id, - 'Type' : type, - 'SpotPrice' : price} - if count: - params['InstanceCount'] = count - if valid_from: - params['ValidFrom'] = valid_from - if valid_until: - params['ValidUntil'] = valid_until - if launch_group: - params['LaunchGroup'] = launch_group - if availability_zone_group: - params['AvailabilityZoneGroup'] = availability_zone_group - if key_name: - params['LaunchSpecification.KeyName'] = key_name - if security_groups: - l = [] - for group in security_groups: - if isinstance(group, SecurityGroup): - l.append(group.name) - else: - l.append(group) - self.build_list_params(params, l, - 'LaunchSpecification.SecurityGroup') - if user_data: - params['LaunchSpecification.UserData'] = base64.b64encode(user_data) - if addressing_type: - params['LaunchSpecification.AddressingType'] = addressing_type - if instance_type: - params['LaunchSpecification.InstanceType'] = instance_type - if placement: - params['LaunchSpecification.Placement.AvailabilityZone'] = placement - if kernel_id: - params['LaunchSpecification.KernelId'] = kernel_id - if ramdisk_id: - params['LaunchSpecification.RamdiskId'] = ramdisk_id - if monitoring_enabled: - params['LaunchSpecification.Monitoring.Enabled'] = 'true' - if subnet_id: - params['LaunchSpecification.SubnetId'] = subnet_id - if block_device_map: - block_device_map.build_list_params(params, 'LaunchSpecification.') - return self.get_list('RequestSpotInstances', params, - [('item', SpotInstanceRequest)], - verb='POST') - - - def cancel_spot_instance_requests(self, request_ids): - """ - Cancel the specified Spot Instance Requests. - - :type request_ids: list - :param request_ids: A list of strings of the Request IDs to terminate - - :rtype: list - :return: A list of the instances terminated - """ - params = {} - if request_ids: - self.build_list_params(params, request_ids, 'SpotInstanceRequestId') - return self.get_list('CancelSpotInstanceRequests', params, - [('item', Instance)], verb='POST') - - def get_spot_datafeed_subscription(self): - """ - Return the current spot instance data feed subscription - associated with this account, if any. - - :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` - :return: The datafeed subscription object or None - """ - return self.get_object('DescribeSpotDatafeedSubscription', - None, SpotDatafeedSubscription, verb='POST') - - def create_spot_datafeed_subscription(self, bucket, prefix): - """ - Create a spot instance datafeed subscription for this account. - - :type bucket: str or unicode - :param bucket: The name of the bucket where spot instance data - will be written. The account issuing this request - must have FULL_CONTROL access to the bucket - specified in the request. - - :type prefix: str or unicode - :param prefix: An optional prefix that will be pre-pended to all - data files written to the bucket. - - :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` - :return: The datafeed subscription object or None - """ - params = {'Bucket' : bucket} - if prefix: - params['Prefix'] = prefix - return self.get_object('CreateSpotDatafeedSubscription', - params, SpotDatafeedSubscription, verb='POST') - - def delete_spot_datafeed_subscription(self): - """ - Delete the current spot instance data feed subscription - associated with this account - - :rtype: bool - :return: True if successful - """ - return self.get_status('DeleteSpotDatafeedSubscription', None, verb='POST') - - # Zone methods - - def get_all_zones(self, zones=None, filters=None): - """ - Get all Availability Zones associated with the current region. - - :type zones: list - :param zones: Optional list of zones. If this list is present, - only the Zones associated with these zone names - will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list of :class:`boto.ec2.zone.Zone` - :return: The requested Zone objects - """ - params = {} - if zones: - self.build_list_params(params, zones, 'ZoneName') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeAvailabilityZones', params, [('item', Zone)], verb='POST') - - # Address methods - - def get_all_addresses(self, addresses=None, filters=None): - """ - Get all EIP's associated with the current credentials. - - :type addresses: list - :param addresses: Optional list of addresses. If this list is present, - only the Addresses associated with these addresses - will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list of :class:`boto.ec2.address.Address` - :return: The requested Address objects - """ - params = {} - if addresses: - self.build_list_params(params, addresses, 'PublicIp') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeAddresses', params, [('item', Address)], verb='POST') - - def allocate_address(self): - """ - Allocate a new Elastic IP address and associate it with your account. - - :rtype: :class:`boto.ec2.address.Address` - :return: The newly allocated Address - """ - return self.get_object('AllocateAddress', {}, Address, verb='POST') - - def associate_address(self, instance_id, public_ip): - """ - Associate an Elastic IP address with a currently running instance. - - :type instance_id: string - :param instance_id: The ID of the instance - - :type public_ip: string - :param public_ip: The public IP address - - :rtype: bool - :return: True if successful - """ - params = {'InstanceId' : instance_id, 'PublicIp' : public_ip} - return self.get_status('AssociateAddress', params, verb='POST') - - def disassociate_address(self, public_ip): - """ - Disassociate an Elastic IP address from a currently running instance. - - :type public_ip: string - :param public_ip: The public IP address - - :rtype: bool - :return: True if successful - """ - params = {'PublicIp' : public_ip} - return self.get_status('DisassociateAddress', params, verb='POST') - - def release_address(self, public_ip): - """ - Free up an Elastic IP address - - :type public_ip: string - :param public_ip: The public IP address - - :rtype: bool - :return: True if successful - """ - params = {'PublicIp' : public_ip} - return self.get_status('ReleaseAddress', params, verb='POST') - - # Volume methods - - def get_all_volumes(self, volume_ids=None, filters=None): - """ - Get all Volumes associated with the current credentials. - - :type volume_ids: list - :param volume_ids: Optional list of volume ids. If this list is present, - only the volumes associated with these volume ids - will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list of :class:`boto.ec2.volume.Volume` - :return: The requested Volume objects - """ - params = {} - if volume_ids: - self.build_list_params(params, volume_ids, 'VolumeId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeVolumes', params, [('item', Volume)], verb='POST') - - def create_volume(self, size, zone, snapshot=None): - """ - Create a new EBS Volume. - - :type size: int - :param size: The size of the new volume, in GiB - - :type zone: string or :class:`boto.ec2.zone.Zone` - :param zone: The availability zone in which the Volume will be created. - - :type snapshot: string or :class:`boto.ec2.snapshot.Snapshot` - :param snapshot: The snapshot from which the new Volume will be created. - """ - if isinstance(zone, Zone): - zone = zone.name - params = {'AvailabilityZone' : zone} - if size: - params['Size'] = size - if snapshot: - if isinstance(snapshot, Snapshot): - snapshot = snapshot.id - params['SnapshotId'] = snapshot - return self.get_object('CreateVolume', params, Volume, verb='POST') - - def delete_volume(self, volume_id): - """ - Delete an EBS volume. - - :type volume_id: str - :param volume_id: The ID of the volume to be delete. - - :rtype: bool - :return: True if successful - """ - params = {'VolumeId': volume_id} - return self.get_status('DeleteVolume', params, verb='POST') - - def attach_volume(self, volume_id, instance_id, device): - """ - Attach an EBS volume to an EC2 instance. - - :type volume_id: str - :param volume_id: The ID of the EBS volume to be attached. - - :type instance_id: str - :param instance_id: The ID of the EC2 instance to which it will - be attached. - - :type device: str - :param device: The device on the instance through which the - volume will be exposted (e.g. /dev/sdh) - - :rtype: bool - :return: True if successful - """ - params = {'InstanceId' : instance_id, - 'VolumeId' : volume_id, - 'Device' : device} - return self.get_status('AttachVolume', params, verb='POST') - - def detach_volume(self, volume_id, instance_id=None, - device=None, force=False): - """ - Detach an EBS volume from an EC2 instance. - - :type volume_id: str - :param volume_id: The ID of the EBS volume to be attached. - - :type instance_id: str - :param instance_id: The ID of the EC2 instance from which it will - be detached. - - :type device: str - :param device: The device on the instance through which the - volume is exposted (e.g. /dev/sdh) - - :type force: bool - :param force: Forces detachment if the previous detachment attempt did - not occur cleanly. This option can lead to data loss or - a corrupted file system. Use this option only as a last - resort to detach a volume from a failed instance. The - instance will not have an opportunity to flush file system - caches nor file system meta data. If you use this option, - you must perform file system check and repair procedures. - - :rtype: bool - :return: True if successful - """ - params = {'VolumeId' : volume_id} - if instance_id: - params['InstanceId'] = instance_id - if device: - params['Device'] = device - if force: - params['Force'] = 'true' - return self.get_status('DetachVolume', params, verb='POST') - - # Snapshot methods - - def get_all_snapshots(self, snapshot_ids=None, - owner=None, restorable_by=None, - filters=None): - """ - Get all EBS Snapshots associated with the current credentials. - - :type snapshot_ids: list - :param snapshot_ids: Optional list of snapshot ids. If this list is - present, only the Snapshots associated with - these snapshot ids will be returned. - - :type owner: str - :param owner: If present, only the snapshots owned by the specified user - will be returned. Valid values are: - - * self - * amazon - * AWS Account ID - - :type restorable_by: str - :param restorable_by: If present, only the snapshots that are restorable - by the specified account id will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list of :class:`boto.ec2.snapshot.Snapshot` - :return: The requested Snapshot objects - """ - params = {} - if snapshot_ids: - self.build_list_params(params, snapshot_ids, 'SnapshotId') - if owner: - params['Owner'] = owner - if restorable_by: - params['RestorableBy'] = restorable_by - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeSnapshots', params, [('item', Snapshot)], verb='POST') - - def create_snapshot(self, volume_id, description=None): - """ - Create a snapshot of an existing EBS Volume. - - :type volume_id: str - :param volume_id: The ID of the volume to be snapshot'ed - - :type description: str - :param description: A description of the snapshot. - Limited to 255 characters. - - :rtype: bool - :return: True if successful - """ - params = {'VolumeId' : volume_id} - if description: - params['Description'] = description[0:255] - snapshot = self.get_object('CreateSnapshot', params, Snapshot, verb='POST') - volume = self.get_all_volumes([volume_id])[0] - volume_name = volume.tags.get('Name') - if volume_name: - snapshot.add_tag('Name', volume_name) - return snapshot - - def delete_snapshot(self, snapshot_id): - params = {'SnapshotId': snapshot_id} - return self.get_status('DeleteSnapshot', params, verb='POST') - - def trim_snapshots(self, hourly_backups = 8, daily_backups = 7, weekly_backups = 4): - """ - Trim excess snapshots, based on when they were taken. More current snapshots are - retained, with the number retained decreasing as you move back in time. - - If ebs volumes have a 'Name' tag with a value, their snapshots will be assigned the same - tag when they are created. The values of the 'Name' tags for snapshots are used by this - function to group snapshots taken from the same volume (or from a series of like-named - volumes over time) for trimming. - - For every group of like-named snapshots, this function retains the newest and oldest - snapshots, as well as, by default, the first snapshots taken in each of the last eight - hours, the first snapshots taken in each of the last seven days, the first snapshots - taken in the last 4 weeks (counting Midnight Sunday morning as the start of the week), - and the first snapshot from the first Sunday of each month forever. - - :type hourly_backups: int - :param hourly_backups: How many recent hourly backups should be saved. - - :type daily_backups: int - :param daily_backups: How many recent daily backups should be saved. - - :type weekly_backups: int - :param weekly_backups: How many recent weekly backups should be saved. - """ - - # This function first builds up an ordered list of target times that snapshots should be saved for - # (last 8 hours, last 7 days, etc.). Then a map of snapshots is constructed, with the keys being - # the snapshot / volume names and the values being arrays of chornologically sorted snapshots. - # Finally, for each array in the map, we go through the snapshot array and the target time array - # in an interleaved fashion, deleting snapshots whose start_times don't immediately follow a - # target time (we delete a snapshot if there's another snapshot that was made closer to the - # preceding target time). - - now = datetime.utcnow() # work with UTC time, which is what the snapshot start time is reported in - last_hour = datetime(now.year, now.month, now.day, now.hour) - last_midnight = datetime(now.year, now.month, now.day) - last_sunday = datetime(now.year, now.month, now.day) - timedelta(days = (now.weekday() + 1) % 7) - start_of_month = datetime(now.year, now.month, 1) - - target_backup_times = [] - - oldest_snapshot_date = datetime(2007, 1, 1) # there are no snapshots older than 1/1/2007 - - for hour in range(0, hourly_backups): - target_backup_times.append(last_hour - timedelta(hours = hour)) - - for day in range(0, daily_backups): - target_backup_times.append(last_midnight - timedelta(days = day)) - - for week in range(0, weekly_backups): - target_backup_times.append(last_sunday - timedelta(weeks = week)) - - one_day = timedelta(days = 1) - while start_of_month > oldest_snapshot_date: - # append the start of the month to the list of snapshot dates to save: - target_backup_times.append(start_of_month) - # there's no timedelta setting for one month, so instead: - # decrement the day by one, so we go to the final day of the previous month... - start_of_month -= one_day - # ... and then go to the first day of that previous month: - start_of_month = datetime(start_of_month.year, start_of_month.month, 1) - - temp = [] - - for t in target_backup_times: - if temp.__contains__(t) == False: - temp.append(t) - - target_backup_times = temp - target_backup_times.reverse() # make the oldest date first - - # get all the snapshots, sort them by date and time, and organize them into one array for each volume: - all_snapshots = self.get_all_snapshots(owner = 'self') - all_snapshots.sort(cmp = lambda x, y: cmp(x.start_time, y.start_time)) # oldest first - snaps_for_each_volume = {} - for snap in all_snapshots: - # the snapshot name and the volume name are the same. The snapshot name is set from the volume - # name at the time the snapshot is taken - volume_name = snap.tags.get('Name') - if volume_name: - # only examine snapshots that have a volume name - snaps_for_volume = snaps_for_each_volume.get(volume_name) - if not snaps_for_volume: - snaps_for_volume = [] - snaps_for_each_volume[volume_name] = snaps_for_volume - snaps_for_volume.append(snap) - - # Do a running comparison of snapshot dates to desired time periods, keeping the oldest snapshot in each - # time period and deleting the rest: - for volume_name in snaps_for_each_volume: - snaps = snaps_for_each_volume[volume_name] - snaps = snaps[:-1] # never delete the newest snapshot, so remove it from consideration - time_period_number = 0 - snap_found_for_this_time_period = False - for snap in snaps: - check_this_snap = True - while check_this_snap and time_period_number < target_backup_times.__len__(): - snap_date = datetime.strptime(snap.start_time, '%Y-%m-%dT%H:%M:%S.000Z') - if snap_date < target_backup_times[time_period_number]: - # the snap date is before the cutoff date. Figure out if it's the first snap in this - # date range and act accordingly (since both date the date ranges and the snapshots - # are sorted chronologically, we know this snapshot isn't in an earlier date range): - if snap_found_for_this_time_period == True: - if not snap.tags.get('preserve_snapshot'): - # as long as the snapshot wasn't marked with the 'preserve_snapshot' tag, delete it: - self.delete_snapshot(snap.id) - boto.log.info('Trimmed snapshot %s (%s)' % (snap.tags['Name'], snap.start_time)) - # go on and look at the next snapshot, leaving the time period alone - else: - # this was the first snapshot found for this time period. Leave it alone and look at the - # next snapshot: - snap_found_for_this_time_period = True - check_this_snap = False - else: - # the snap is after the cutoff date. Check it against the next cutoff date - time_period_number += 1 - snap_found_for_this_time_period = False - - - def get_snapshot_attribute(self, snapshot_id, - attribute='createVolumePermission'): - """ - Get information about an attribute of a snapshot. Only one attribute - can be specified per call. - - :type snapshot_id: str - :param snapshot_id: The ID of the snapshot. - - :type attribute: str - :param attribute: The requested attribute. Valid values are: - - * createVolumePermission - - :rtype: list of :class:`boto.ec2.snapshotattribute.SnapshotAttribute` - :return: The requested Snapshot attribute - """ - params = {'Attribute' : attribute} - if snapshot_id: - params['SnapshotId'] = snapshot_id - return self.get_object('DescribeSnapshotAttribute', params, - SnapshotAttribute, verb='POST') - - def modify_snapshot_attribute(self, snapshot_id, - attribute='createVolumePermission', - operation='add', user_ids=None, groups=None): - """ - Changes an attribute of an image. - - :type snapshot_id: string - :param snapshot_id: The snapshot id you wish to change - - :type attribute: string - :param attribute: The attribute you wish to change. Valid values are: - createVolumePermission - - :type operation: string - :param operation: Either add or remove (this is required for changing - snapshot ermissions) - - :type user_ids: list - :param user_ids: The Amazon IDs of users to add/remove attributes - - :type groups: list - :param groups: The groups to add/remove attributes. The only valid - value at this time is 'all'. - - """ - params = {'SnapshotId' : snapshot_id, - 'Attribute' : attribute, - 'OperationType' : operation} - if user_ids: - self.build_list_params(params, user_ids, 'UserId') - if groups: - self.build_list_params(params, groups, 'UserGroup') - return self.get_status('ModifySnapshotAttribute', params, verb='POST') - - def reset_snapshot_attribute(self, snapshot_id, - attribute='createVolumePermission'): - """ - Resets an attribute of a snapshot to its default value. - - :type snapshot_id: string - :param snapshot_id: ID of the snapshot - - :type attribute: string - :param attribute: The attribute to reset - - :rtype: bool - :return: Whether the operation succeeded or not - """ - params = {'SnapshotId' : snapshot_id, - 'Attribute' : attribute} - return self.get_status('ResetSnapshotAttribute', params, verb='POST') - - # Keypair methods - - def get_all_key_pairs(self, keynames=None, filters=None): - """ - Get all key pairs associated with your account. - - :type keynames: list - :param keynames: A list of the names of keypairs to retrieve. - If not provided, all key pairs will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.keypair.KeyPair` - """ - params = {} - if keynames: - self.build_list_params(params, keynames, 'KeyName') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeKeyPairs', params, [('item', KeyPair)], verb='POST') - - def get_key_pair(self, keyname): - """ - Convenience method to retrieve a specific keypair (KeyPair). - - :type image_id: string - :param image_id: the ID of the Image to retrieve - - :rtype: :class:`boto.ec2.keypair.KeyPair` - :return: The KeyPair specified or None if it is not found - """ - try: - return self.get_all_key_pairs(keynames=[keyname])[0] - except IndexError: # None of those key pairs available - return None - - def create_key_pair(self, key_name): - """ - Create a new key pair for your account. - This will create the key pair within the region you - are currently connected to. - - :type key_name: string - :param key_name: The name of the new keypair - - :rtype: :class:`boto.ec2.keypair.KeyPair` - :return: The newly created :class:`boto.ec2.keypair.KeyPair`. - The material attribute of the new KeyPair object - will contain the the unencrypted PEM encoded RSA private key. - """ - params = {'KeyName':key_name} - return self.get_object('CreateKeyPair', params, KeyPair, verb='POST') - - def delete_key_pair(self, key_name): - """ - Delete a key pair from your account. - - :type key_name: string - :param key_name: The name of the keypair to delete - """ - params = {'KeyName':key_name} - return self.get_status('DeleteKeyPair', params, verb='POST') - - def import_key_pair(self, key_name, public_key_material): - """ - mports the public key from an RSA key pair that you created - with a third-party tool. - - Supported formats: - - * OpenSSH public key format (e.g., the format - in ~/.ssh/authorized_keys) - - * Base64 encoded DER format - - * SSH public key file format as specified in RFC4716 - - DSA keys are not supported. Make sure your key generator is - set up to create RSA keys. - - Supported lengths: 1024, 2048, and 4096. - - :type key_name: string - :param key_name: The name of the new keypair - - :type public_key_material: string - :param public_key_material: The public key. You must base64 encode - the public key material before sending - it to AWS. - - :rtype: :class:`boto.ec2.keypair.KeyPair` - :return: The newly created :class:`boto.ec2.keypair.KeyPair`. - The material attribute of the new KeyPair object - will contain the the unencrypted PEM encoded RSA private key. - """ - params = {'KeyName' : key_name, - 'PublicKeyMaterial' : public_key_material} - return self.get_object('ImportKeyPair', params, KeyPair, verb='POST') - - # SecurityGroup methods - - def get_all_security_groups(self, groupnames=None, filters=None): - """ - Get all security groups associated with your account in a region. - - :type groupnames: list - :param groupnames: A list of the names of security groups to retrieve. - If not provided, all security groups will be - returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.securitygroup.SecurityGroup` - """ - params = {} - if groupnames: - self.build_list_params(params, groupnames, 'GroupName') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeSecurityGroups', params, - [('item', SecurityGroup)], verb='POST') - - def create_security_group(self, name, description): - """ - Create a new security group for your account. - This will create the security group within the region you - are currently connected to. - - :type name: string - :param name: The name of the new security group - - :type description: string - :param description: The description of the new security group - - :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` - :return: The newly created :class:`boto.ec2.keypair.KeyPair`. - """ - params = {'GroupName':name, 'GroupDescription':description} - group = self.get_object('CreateSecurityGroup', params, SecurityGroup, verb='POST') - group.name = name - group.description = description - return group - - def delete_security_group(self, name): - """ - Delete a security group from your account. - - :type key_name: string - :param key_name: The name of the keypair to delete - """ - params = {'GroupName':name} - return self.get_status('DeleteSecurityGroup', params, verb='POST') - - def _authorize_deprecated(self, group_name, src_security_group_name=None, - src_security_group_owner_id=None): - """ - This method is called only when someone tries to authorize a group - without specifying a from_port or to_port. Until recently, that was - the only way to do group authorization but the EC2 API has been - changed to now require a from_port and to_port when specifying a - group. This is a much better approach but I don't want to break - existing boto applications that depend on the old behavior, hence - this kludge. - - :type group_name: string - :param group_name: The name of the security group you are adding - the rule to. - - :type src_security_group_name: string - :param src_security_group_name: The name of the security group you are - granting access to. - - :type src_security_group_owner_id: string - :param src_security_group_owner_id: The ID of the owner of the security - group you are granting access to. - - :rtype: bool - :return: True if successful. - """ - warnings.warn('FromPort and ToPort now required for group authorization', - DeprecationWarning) - params = {'GroupName':group_name} - if src_security_group_name: - params['SourceSecurityGroupName'] = src_security_group_name - if src_security_group_owner_id: - params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id - return self.get_status('AuthorizeSecurityGroupIngress', params, verb='POST') - - def authorize_security_group(self, group_name, src_security_group_name=None, - src_security_group_owner_id=None, - ip_protocol=None, from_port=None, to_port=None, - cidr_ip=None): - """ - Add a new rule to an existing security group. - You need to pass in either src_security_group_name and - src_security_group_owner_id OR ip_protocol, from_port, to_port, - and cidr_ip. In other words, either you are authorizing another - group or you are authorizing some ip-based rule. - - :type group_name: string - :param group_name: The name of the security group you are adding - the rule to. - - :type src_security_group_name: string - :param src_security_group_name: The name of the security group you are - granting access to. - - :type src_security_group_owner_id: string - :param src_security_group_owner_id: The ID of the owner of the security - group you are granting access to. - - :type ip_protocol: string - :param ip_protocol: Either tcp | udp | icmp - - :type from_port: int - :param from_port: The beginning port number you are enabling - - :type to_port: int - :param to_port: The ending port number you are enabling - - :type cidr_ip: string - :param cidr_ip: The CIDR block you are providing access to. - See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - - :rtype: bool - :return: True if successful. - """ - if src_security_group_name: - if from_port is None and to_port is None and ip_protocol is None: - return self._authorize_deprecated(group_name, - src_security_group_name, - src_security_group_owner_id) - params = {'GroupName':group_name} - if src_security_group_name: - params['IpPermissions.1.Groups.1.GroupName'] = src_security_group_name - if src_security_group_owner_id: - params['IpPermissions.1.Groups.1.UserId'] = src_security_group_owner_id - if ip_protocol: - params['IpPermissions.1.IpProtocol'] = ip_protocol - if from_port: - params['IpPermissions.1.FromPort'] = from_port - if to_port: - params['IpPermissions.1.ToPort'] = to_port - if cidr_ip: - params['IpPermissions.1.IpRanges.1.CidrIp'] = cidr_ip - return self.get_status('AuthorizeSecurityGroupIngress', params, verb='POST') - - def _revoke_deprecated(self, group_name, src_security_group_name=None, - src_security_group_owner_id=None): - """ - This method is called only when someone tries to revoke a group - without specifying a from_port or to_port. Until recently, that was - the only way to do group revocation but the EC2 API has been - changed to now require a from_port and to_port when specifying a - group. This is a much better approach but I don't want to break - existing boto applications that depend on the old behavior, hence - this kludge. - - :type group_name: string - :param group_name: The name of the security group you are adding - the rule to. - - :type src_security_group_name: string - :param src_security_group_name: The name of the security group you are - granting access to. - - :type src_security_group_owner_id: string - :param src_security_group_owner_id: The ID of the owner of the security - group you are granting access to. - - :rtype: bool - :return: True if successful. - """ - warnings.warn('FromPort and ToPort now required for group authorization', - DeprecationWarning) - params = {'GroupName':group_name} - if src_security_group_name: - params['SourceSecurityGroupName'] = src_security_group_name - if src_security_group_owner_id: - params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id - return self.get_status('RevokeSecurityGroupIngress', params, verb='POST') - - def revoke_security_group(self, group_name, src_security_group_name=None, - src_security_group_owner_id=None, - ip_protocol=None, from_port=None, to_port=None, - cidr_ip=None): - """ - Remove an existing rule from an existing security group. - You need to pass in either src_security_group_name and - src_security_group_owner_id OR ip_protocol, from_port, to_port, - and cidr_ip. In other words, either you are revoking another - group or you are revoking some ip-based rule. - - :type group_name: string - :param group_name: The name of the security group you are removing - the rule from. - - :type src_security_group_name: string - :param src_security_group_name: The name of the security group you are - revoking access to. - - :type src_security_group_owner_id: string - :param src_security_group_owner_id: The ID of the owner of the security - group you are revoking access to. - - :type ip_protocol: string - :param ip_protocol: Either tcp | udp | icmp - - :type from_port: int - :param from_port: The beginning port number you are disabling - - :type to_port: int - :param to_port: The ending port number you are disabling - - :type cidr_ip: string - :param cidr_ip: The CIDR block you are revoking access to. - See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - - :rtype: bool - :return: True if successful. - """ - if src_security_group_name: - if from_port is None and to_port is None and ip_protocol is None: - return self._revoke_deprecated(group_name, - src_security_group_name, - src_security_group_owner_id) - params = {'GroupName':group_name} - if src_security_group_name: - params['IpPermissions.1.Groups.1.GroupName'] = src_security_group_name - if src_security_group_owner_id: - params['IpPermissions.1.Groups.1.UserId'] = src_security_group_owner_id - if ip_protocol: - params['IpPermissions.1.IpProtocol'] = ip_protocol - if from_port: - params['IpPermissions.1.FromPort'] = from_port - if to_port: - params['IpPermissions.1.ToPort'] = to_port - if cidr_ip: - params['IpPermissions.1.IpRanges.1.CidrIp'] = cidr_ip - return self.get_status('RevokeSecurityGroupIngress', params, verb='POST') - - # - # Regions - # - - def get_all_regions(self, region_names=None, filters=None): - """ - Get all available regions for the EC2 service. - - :type region_names: list of str - :param region_names: Names of regions to limit output - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` - """ - params = {} - if region_names: - self.build_list_params(params, region_names, 'RegionName') - if filters: - self.build_filter_params(params, filters) - regions = self.get_list('DescribeRegions', params, [('item', RegionInfo)], verb='POST') - for region in regions: - region.connection_cls = EC2Connection - return regions - - # - # Reservation methods - # - - def get_all_reserved_instances_offerings(self, reserved_instances_id=None, - instance_type=None, - availability_zone=None, - product_description=None, - filters=None): - """ - Describes Reserved Instance offerings that are available for purchase. - - :type reserved_instances_id: str - :param reserved_instances_id: Displays Reserved Instances with the - specified offering IDs. - - :type instance_type: str - :param instance_type: Displays Reserved Instances of the specified - instance type. - - :type availability_zone: str - :param availability_zone: Displays Reserved Instances within the - specified Availability Zone. - - :type product_description: str - :param product_description: Displays Reserved Instances with the - specified product description. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstancesOffering` - """ - params = {} - if reserved_instances_id: - params['ReservedInstancesId'] = reserved_instances_id - if instance_type: - params['InstanceType'] = instance_type - if availability_zone: - params['AvailabilityZone'] = availability_zone - if product_description: - params['ProductDescription'] = product_description - if filters: - self.build_filter_params(params, filters) - - return self.get_list('DescribeReservedInstancesOfferings', - params, [('item', ReservedInstancesOffering)], verb='POST') - - def get_all_reserved_instances(self, reserved_instances_id=None, - filters=None): - """ - Describes Reserved Instance offerings that are available for purchase. - - :type reserved_instance_ids: list - :param reserved_instance_ids: A list of the reserved instance ids that - will be returned. If not provided, all - reserved instances will be returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstance` - """ - params = {} - if reserved_instances_id: - self.build_list_params(params, reserved_instances_id, - 'ReservedInstancesId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeReservedInstances', - params, [('item', ReservedInstance)], verb='POST') - - def purchase_reserved_instance_offering(self, reserved_instances_offering_id, - instance_count=1): - """ - Purchase a Reserved Instance for use with your account. - ** CAUTION ** - This request can result in large amounts of money being charged to your - AWS account. Use with caution! - - :type reserved_instances_offering_id: string - :param reserved_instances_offering_id: The offering ID of the Reserved - Instance to purchase - - :type instance_count: int - :param instance_count: The number of Reserved Instances to purchase. - Default value is 1. - - :rtype: :class:`boto.ec2.reservedinstance.ReservedInstance` - :return: The newly created Reserved Instance - """ - params = {'ReservedInstancesOfferingId' : reserved_instances_offering_id, - 'InstanceCount' : instance_count} - return self.get_object('PurchaseReservedInstancesOffering', params, - ReservedInstance, verb='POST') - - # - # Monitoring - # - - def monitor_instances(self, instance_ids): - """ - Enable CloudWatch monitoring for the supplied instances. - - :type instance_id: list of strings - :param instance_id: The instance ids - - :rtype: list - :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` - """ - params = {} - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_list('MonitorInstances', params, - [('item', InstanceInfo)], verb='POST') - - def monitor_instance(self, instance_id): - """ - Deprecated Version, maintained for backward compatibility. - Enable CloudWatch monitoring for the supplied instance. - - :type instance_id: string - :param instance_id: The instance id - - :rtype: list - :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` - """ - return self.monitor_instances([instance_id]) - - def unmonitor_instance(self, instance_ids): - """ - Disable CloudWatch monitoring for the supplied instance. - - :type instance_id: list of string - :param instance_id: The instance id - - :rtype: list - :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` - """ - params = {} - self.build_list_params(params, instance_ids, 'InstanceId') - return self.get_list('UnmonitorInstances', params, - [('item', InstanceInfo)], verb='POST') - - def unmonitor_instance(self, instance_id): - """ - Deprecated Version, maintained for backward compatibility. - Disable CloudWatch monitoring for the supplied instance. - - :type instance_id: string - :param instance_id: The instance id - - :rtype: list - :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` - """ - return self.unmonitor_instances([instance_id]) - - # - # Bundle Windows Instances - # - - def bundle_instance(self, instance_id, - s3_bucket, - s3_prefix, - s3_upload_policy): - """ - Bundle Windows instance. - - :type instance_id: string - :param instance_id: The instance id - - :type s3_bucket: string - :param s3_bucket: The bucket in which the AMI should be stored. - - :type s3_prefix: string - :param s3_prefix: The beginning of the file name for the AMI. - - :type s3_upload_policy: string - :param s3_upload_policy: Base64 encoded policy that specifies condition - and permissions for Amazon EC2 to upload the - user's image into Amazon S3. - """ - - params = {'InstanceId' : instance_id, - 'Storage.S3.Bucket' : s3_bucket, - 'Storage.S3.Prefix' : s3_prefix, - 'Storage.S3.UploadPolicy' : s3_upload_policy} - s3auth = boto.auth.get_auth_handler(None, boto.config, - self.provider, ['s3']) - params['Storage.S3.AWSAccessKeyId'] = self.aws_access_key_id - signature = s3auth.sign_string(s3_upload_policy) - params['Storage.S3.UploadPolicySignature'] = signature - return self.get_object('BundleInstance', params, - BundleInstanceTask, verb='POST') - - def get_all_bundle_tasks(self, bundle_ids=None, filters=None): - """ - Retrieve current bundling tasks. If no bundle id is specified, all - tasks are retrieved. - - :type bundle_ids: list - :param bundle_ids: A list of strings containing identifiers for - previously created bundling tasks. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - """ - - params = {} - if bundle_ids: - self.build_list_params(params, bundle_ids, 'BundleId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeBundleTasks', params, - [('item', BundleInstanceTask)], verb='POST') - - def cancel_bundle_task(self, bundle_id): - """ - Cancel a previously submitted bundle task - - :type bundle_id: string - :param bundle_id: The identifier of the bundle task to cancel. - """ - - params = {'BundleId' : bundle_id} - return self.get_object('CancelBundleTask', params, - BundleInstanceTask, verb='POST') - - def get_password_data(self, instance_id): - """ - Get encrypted administrator password for a Windows instance. - - :type instance_id: string - :param instance_id: The identifier of the instance to retrieve the - password for. - """ - - params = {'InstanceId' : instance_id} - rs = self.get_object('GetPasswordData', params, ResultSet, verb='POST') - return rs.passwordData - - # - # Cluster Placement Groups - # - - def get_all_placement_groups(self, groupnames=None, filters=None): - """ - Get all placement groups associated with your account in a region. - - :type groupnames: list - :param groupnames: A list of the names of placement groups to retrieve. - If not provided, all placement groups will be - returned. - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: list - :return: A list of :class:`boto.ec2.placementgroup.PlacementGroup` - """ - params = {} - if groupnames: - self.build_list_params(params, groupnames, 'GroupName') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribePlacementGroups', params, - [('item', PlacementGroup)], verb='POST') - - def create_placement_group(self, name, strategy='cluster'): - """ - Create a new placement group for your account. - This will create the placement group within the region you - are currently connected to. - - :type name: string - :param name: The name of the new placement group - - :type strategy: string - :param strategy: The placement strategy of the new placement group. - Currently, the only acceptable value is "cluster". - - :rtype: :class:`boto.ec2.placementgroup.PlacementGroup` - :return: The newly created :class:`boto.ec2.keypair.KeyPair`. - """ - params = {'GroupName':name, 'Strategy':strategy} - group = self.get_status('CreatePlacementGroup', params, verb='POST') - return group - - def delete_placement_group(self, name): - """ - Delete a placement group from your account. - - :type key_name: string - :param key_name: The name of the keypair to delete - """ - params = {'GroupName':name} - return self.get_status('DeletePlacementGroup', params, verb='POST') - - # Tag methods - - def build_tag_param_list(self, params, tags): - keys = tags.keys() - keys.sort() - i = 1 - for key in keys: - value = tags[key] - params['Tag.%d.Key'%i] = key - if value is None: - value = '' - params['Tag.%d.Value'%i] = value - i += 1 - - def get_all_tags(self, tags=None, filters=None): - """ - Retrieve all the metadata tags associated with your account. - - :type tags: list - :param tags: A list of mumble - - :type filters: dict - :param filters: Optional filters that can be used to limit - the results returned. Filters are provided - in the form of a dictionary consisting of - filter names as the key and filter values - as the value. The set of allowable filter - names/values is dependent on the request - being performed. Check the EC2 API guide - for details. - - :rtype: dict - :return: A dictionary containing metadata tags - """ - params = {} - if tags: - self.build_list_params(params, instance_ids, 'InstanceId') - if filters: - self.build_filter_params(params, filters) - return self.get_list('DescribeTags', params, [('item', Tag)], verb='POST') - - def create_tags(self, resource_ids, tags): - """ - Create new metadata tags for the specified resource ids. - - :type resource_ids: list - :param resource_ids: List of strings - - :type tags: dict - :param tags: A dictionary containing the name/value pairs - - """ - params = {} - self.build_list_params(params, resource_ids, 'ResourceId') - self.build_tag_param_list(params, tags) - return self.get_status('CreateTags', params, verb='POST') - - def delete_tags(self, resource_ids, tags): - """ - Delete metadata tags for the specified resource ids. - - :type resource_ids: list - :param resource_ids: List of strings - - :type tags: dict or list - :param tags: Either a dictionary containing name/value pairs - or a list containing just tag names. - If you pass in a dictionary, the values must - match the actual tag values or the tag will - not be deleted. - - """ - if isinstance(tags, list): - tags = {}.fromkeys(tags, None) - params = {} - self.build_list_params(params, resource_ids, 'ResourceId') - self.build_tag_param_list(params, tags) - return self.get_status('DeleteTags', params, verb='POST') - diff --git a/backup/src/boto/ec2/ec2object.py b/backup/src/boto/ec2/ec2object.py deleted file mode 100644 index 6e37596..0000000 --- a/backup/src/boto/ec2/ec2object.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Object -""" -from boto.ec2.tag import TagSet - -class EC2Object(object): - - def __init__(self, connection=None): - self.connection = connection - if self.connection and hasattr(self.connection, 'region'): - self.region = connection.region - else: - self.region = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - setattr(self, name, value) - - -class TaggedEC2Object(EC2Object): - """ - Any EC2 resource that can be tagged should be represented - by a Python object that subclasses this class. This class - has the mechanism in place to handle the tagSet element in - the Describe* responses. If tags are found, it will create - a TagSet object and allow it to parse and collect the tags - into a dict that is stored in the "tags" attribute of the - object. - """ - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.tags = TagSet() - - def startElement(self, name, attrs, connection): - if name == 'tagSet': - return self.tags - else: - return None - - def add_tag(self, key, value=None): - """ - Add a tag to this object. Tag's are stored by AWS and can be used - to organize and filter resources. Adding a tag involves a round-trip - to the EC2 service. - - :type key: str - :param key: The key or name of the tag being stored. - - :type value: str - :param value: An optional value that can be stored with the tag. - """ - status = self.connection.create_tags([self.id], {key : value}) - if self.tags is None: - self.tags = TagSet() - self.tags[key] = value - - def remove_tag(self, key, value=None): - """ - Remove a tag from this object. Removing a tag involves a round-trip - to the EC2 service. - - :type key: str - :param key: The key or name of the tag being stored. - - :type value: str - :param value: An optional value that can be stored with the tag. - If a value is provided, it must match the value - currently stored in EC2. If not, the tag will not - be removed. - """ - if value: - tags = {key : value} - else: - tags = [key] - status = self.connection.delete_tags([self.id], tags) - if key in self.tags: - del self.tags[key] diff --git a/backup/src/boto/ec2/elb/__init__.py b/backup/src/boto/ec2/elb/__init__.py deleted file mode 100644 index f4061d3..0000000 --- a/backup/src/boto/ec2/elb/__init__.py +++ /dev/null @@ -1,427 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -""" -This module provides an interface to the Elastic Compute Cloud (EC2) -load balancing service from AWS. -""" -from boto.connection import AWSQueryConnection -from boto.ec2.instanceinfo import InstanceInfo -from boto.ec2.elb.loadbalancer import LoadBalancer -from boto.ec2.elb.instancestate import InstanceState -from boto.ec2.elb.healthcheck import HealthCheck -from boto.regioninfo import RegionInfo -import boto - -RegionData = { - 'us-east-1' : 'elasticloadbalancing.us-east-1.amazonaws.com', - 'us-west-1' : 'elasticloadbalancing.us-west-1.amazonaws.com', - 'eu-west-1' : 'elasticloadbalancing.eu-west-1.amazonaws.com', - 'ap-southeast-1' : 'elasticloadbalancing.ap-southeast-1.amazonaws.com'} - -def regions(): - """ - Get all available regions for the SDB service. - - :rtype: list - :return: A list of :class:`boto.RegionInfo` instances - """ - regions = [] - for region_name in RegionData: - region = RegionInfo(name=region_name, - endpoint=RegionData[region_name], - connection_cls=ELBConnection) - regions.append(region) - return regions - -def connect_to_region(region_name, **kw_params): - """ - Given a valid region name, return a - :class:`boto.ec2.elb.ELBConnection`. - - :param str region_name: The name of the region to connect to. - - :rtype: :class:`boto.ec2.ELBConnection` or ``None`` - :return: A connection to the given region, or None if an invalid region - name is given - """ - for region in regions(): - if region.name == region_name: - return region.connect(**kw_params) - return None - -class ELBConnection(AWSQueryConnection): - - APIVersion = boto.config.get('Boto', 'elb_version', '2010-07-01') - DefaultRegionName = boto.config.get('Boto', 'elb_region_name', 'us-east-1') - DefaultRegionEndpoint = boto.config.get('Boto', 'elb_region_endpoint', - 'elasticloadbalancing.amazonaws.com') - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=False, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - """ - Init method to create a new connection to EC2 Load Balancing Service. - - B{Note:} The region argument is overridden by the region specified in - the boto configuration file. - """ - if not region: - region = RegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['ec2'] - - def build_list_params(self, params, items, label): - if isinstance(items, str): - items = [items] - for i in range(1, len(items)+1): - params[label % i] = items[i-1] - - def get_all_load_balancers(self, load_balancer_names=None): - """ - Retrieve all load balancers associated with your account. - - :type load_balancer_names: list - :param load_balancer_names: An optional list of load balancer names - - :rtype: list - :return: A list of :class:`boto.ec2.elb.loadbalancer.LoadBalancer` - """ - params = {} - if load_balancer_names: - self.build_list_params(params, load_balancer_names, 'LoadBalancerNames.member.%d') - return self.get_list('DescribeLoadBalancers', params, [('member', LoadBalancer)]) - - - def create_load_balancer(self, name, zones, listeners): - """ - Create a new load balancer for your account. - - :type name: string - :param name: The mnemonic name associated with the new load balancer - - :type zones: List of strings - :param zones: The names of the availability zone(s) to add. - - :type listeners: List of tuples - :param listeners: Each tuple contains three or four values, - (LoadBalancerPortNumber, InstancePortNumber, Protocol, - [SSLCertificateId]) - where LoadBalancerPortNumber and InstancePortNumber are - integer values between 1 and 65535, Protocol is a - string containing either 'TCP', 'HTTP' or 'HTTPS'; - SSLCertificateID is the ARN of a AWS AIM certificate, - and must be specified when doing HTTPS. - - :rtype: :class:`boto.ec2.elb.loadbalancer.LoadBalancer` - :return: The newly created :class:`boto.ec2.elb.loadbalancer.LoadBalancer` - """ - params = {'LoadBalancerName' : name} - for i in range(0, len(listeners)): - params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] - params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] - params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] - if listeners[i][2]=='HTTPS': - params['Listeners.member.%d.SSLCertificateId' % (i+1)] = listeners[i][3] - self.build_list_params(params, zones, 'AvailabilityZones.member.%d') - load_balancer = self.get_object('CreateLoadBalancer', params, LoadBalancer) - load_balancer.name = name - load_balancer.listeners = listeners - load_balancer.availability_zones = zones - return load_balancer - - def create_load_balancer_listeners(self, name, listeners): - """ - Creates a Listener (or group of listeners) for an existing Load Balancer - - :type name: string - :param name: The name of the load balancer to create the listeners for - - :type listeners: List of tuples - :param listeners: Each tuple contains three values, - (LoadBalancerPortNumber, InstancePortNumber, Protocol, - [SSLCertificateId]) - where LoadBalancerPortNumber and InstancePortNumber are - integer values between 1 and 65535, Protocol is a - string containing either 'TCP', 'HTTP' or 'HTTPS'; - SSLCertificateID is the ARN of a AWS AIM certificate, - and must be specified when doing HTTPS. - - :return: The status of the request - """ - params = {'LoadBalancerName' : name} - for i in range(0, len(listeners)): - params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] - params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] - params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] - if listeners[i][2]=='HTTPS': - params['Listeners.member.%d.SSLCertificateId' % (i+1)] = listeners[i][3] - return self.get_status('CreateLoadBalancerListeners', params) - - - def delete_load_balancer(self, name): - """ - Delete a Load Balancer from your account. - - :type name: string - :param name: The name of the Load Balancer to delete - """ - params = {'LoadBalancerName': name} - return self.get_status('DeleteLoadBalancer', params) - - def delete_load_balancer_listeners(self, name, ports): - """ - Deletes a load balancer listener (or group of listeners) - - :type name: string - :param name: The name of the load balancer to create the listeners for - - :type ports: List int - :param ports: Each int represents the port on the ELB to be removed - - :return: The status of the request - """ - params = {'LoadBalancerName' : name} - for i in range(0, len(ports)): - params['LoadBalancerPorts.member.%d' % (i+1)] = ports[i] - return self.get_status('DeleteLoadBalancerListeners', params) - - - - def enable_availability_zones(self, load_balancer_name, zones_to_add): - """ - Add availability zones to an existing Load Balancer - All zones must be in the same region as the Load Balancer - Adding zones that are already registered with the Load Balancer - has no effect. - - :type load_balancer_name: string - :param load_balancer_name: The name of the Load Balancer - - :type zones: List of strings - :param zones: The name of the zone(s) to add. - - :rtype: List of strings - :return: An updated list of zones for this Load Balancer. - - """ - params = {'LoadBalancerName' : load_balancer_name} - self.build_list_params(params, zones_to_add, 'AvailabilityZones.member.%d') - return self.get_list('EnableAvailabilityZonesForLoadBalancer', params, None) - - def disable_availability_zones(self, load_balancer_name, zones_to_remove): - """ - Remove availability zones from an existing Load Balancer. - All zones must be in the same region as the Load Balancer. - Removing zones that are not registered with the Load Balancer - has no effect. - You cannot remove all zones from an Load Balancer. - - :type load_balancer_name: string - :param load_balancer_name: The name of the Load Balancer - - :type zones: List of strings - :param zones: The name of the zone(s) to remove. - - :rtype: List of strings - :return: An updated list of zones for this Load Balancer. - - """ - params = {'LoadBalancerName' : load_balancer_name} - self.build_list_params(params, zones_to_remove, 'AvailabilityZones.member.%d') - return self.get_list('DisableAvailabilityZonesForLoadBalancer', params, None) - - def register_instances(self, load_balancer_name, instances): - """ - Add new Instances to an existing Load Balancer. - - :type load_balancer_name: string - :param load_balancer_name: The name of the Load Balancer - - :type instances: List of strings - :param instances: The instance ID's of the EC2 instances to add. - - :rtype: List of strings - :return: An updated list of instances for this Load Balancer. - - """ - params = {'LoadBalancerName' : load_balancer_name} - self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') - return self.get_list('RegisterInstancesWithLoadBalancer', params, [('member', InstanceInfo)]) - - def deregister_instances(self, load_balancer_name, instances): - """ - Remove Instances from an existing Load Balancer. - - :type load_balancer_name: string - :param load_balancer_name: The name of the Load Balancer - - :type instances: List of strings - :param instances: The instance ID's of the EC2 instances to remove. - - :rtype: List of strings - :return: An updated list of instances for this Load Balancer. - - """ - params = {'LoadBalancerName' : load_balancer_name} - self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') - return self.get_list('DeregisterInstancesFromLoadBalancer', params, [('member', InstanceInfo)]) - - def describe_instance_health(self, load_balancer_name, instances=None): - """ - Get current state of all Instances registered to an Load Balancer. - - :type load_balancer_name: string - :param load_balancer_name: The name of the Load Balancer - - :type instances: List of strings - :param instances: The instance ID's of the EC2 instances - to return status for. If not provided, - the state of all instances will be returned. - - :rtype: List of :class:`boto.ec2.elb.instancestate.InstanceState` - :return: list of state info for instances in this Load Balancer. - - """ - params = {'LoadBalancerName' : load_balancer_name} - if instances: - self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') - return self.get_list('DescribeInstanceHealth', params, [('member', InstanceState)]) - - def configure_health_check(self, name, health_check): - """ - Define a health check for the EndPoints. - - :type name: string - :param name: The mnemonic name associated with the new access point - - :type health_check: :class:`boto.ec2.elb.healthcheck.HealthCheck` - :param health_check: A HealthCheck object populated with the desired - values. - - :rtype: :class:`boto.ec2.elb.healthcheck.HealthCheck` - :return: The updated :class:`boto.ec2.elb.healthcheck.HealthCheck` - """ - params = {'LoadBalancerName' : name, - 'HealthCheck.Timeout' : health_check.timeout, - 'HealthCheck.Target' : health_check.target, - 'HealthCheck.Interval' : health_check.interval, - 'HealthCheck.UnhealthyThreshold' : health_check.unhealthy_threshold, - 'HealthCheck.HealthyThreshold' : health_check.healthy_threshold} - return self.get_object('ConfigureHealthCheck', params, HealthCheck) - - def set_lb_listener_SSL_certificate(self, lb_name, lb_port, ssl_certificate_id): - """ - Sets the certificate that terminates the specified listener's SSL - connections. The specified certificate replaces any prior certificate - that was used on the same LoadBalancer and port. - """ - params = { - 'LoadBalancerName' : lb_name, - 'LoadBalancerPort' : lb_port, - 'SSLCertificateId' : ssl_certificate_id, - } - return self.get_status('SetLoadBalancerListenerSSLCertificate', params) - - def create_app_cookie_stickiness_policy(self, name, lb_name, policy_name): - """ - Generates a stickiness policy with sticky session lifetimes that follow - that of an application-generated cookie. This policy can only be - associated with HTTP listeners. - - This policy is similar to the policy created by - CreateLBCookieStickinessPolicy, except that the lifetime of the special - Elastic Load Balancing cookie follows the lifetime of the - application-generated cookie specified in the policy configuration. The - load balancer only inserts a new stickiness cookie when the application - response includes a new application cookie. - - If the application cookie is explicitly removed or expires, the session - stops being sticky until a new application cookie is issued. - """ - params = { - 'CookieName' : name, - 'LoadBalancerName' : lb_name, - 'PolicyName' : policy_name, - } - return self.get_status('CreateAppCookieStickinessPolicy', params) - - def create_lb_cookie_stickiness_policy(self, cookie_expiration_period, lb_name, policy_name): - """ - Generates a stickiness policy with sticky session lifetimes controlled - by the lifetime of the browser (user-agent) or a specified expiration - period. This policy can only be associated only with HTTP listeners. - - When a load balancer implements this policy, the load balancer uses a - special cookie to track the backend server instance for each request. - When the load balancer receives a request, it first checks to see if - this cookie is present in the request. If so, the load balancer sends - the request to the application server specified in the cookie. If not, - the load balancer sends the request to a server that is chosen based on - the existing load balancing algorithm. - - A cookie is inserted into the response for binding subsequent requests - from the same user to that server. The validity of the cookie is based - on the cookie expiration time, which is specified in the policy - configuration. - """ - params = { - 'CookieExpirationPeriod' : cookie_expiration_period, - 'LoadBalancerName' : lb_name, - 'PolicyName' : policy_name, - } - return self.get_status('CreateLBCookieStickinessPolicy', params) - - def delete_lb_policy(self, lb_name, policy_name): - """ - Deletes a policy from the LoadBalancer. The specified policy must not - be enabled for any listeners. - """ - params = { - 'LoadBalancerName' : lb_name, - 'PolicyName' : policy_name, - } - return self.get_status('DeleteLoadBalancerPolicy', params) - - def set_lb_policies_of_listener(self, lb_name, lb_port, policies): - """ - Associates, updates, or disables a policy with a listener on the load - balancer. Currently only zero (0) or one (1) policy can be associated - with a listener. - """ - params = { - 'LoadBalancerName' : lb_name, - 'LoadBalancerPort' : lb_port, - } - self.build_list_params(params, policies, 'PolicyNames.member.%d') - return self.get_status('SetLoadBalancerPoliciesOfListener', params) - - diff --git a/backup/src/boto/ec2/elb/healthcheck.py b/backup/src/boto/ec2/elb/healthcheck.py deleted file mode 100644 index 5a3edbc..0000000 --- a/backup/src/boto/ec2/elb/healthcheck.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class HealthCheck(object): - """ - Represents an EC2 Access Point Health Check - """ - - def __init__(self, access_point=None, interval=30, target=None, - healthy_threshold=3, timeout=5, unhealthy_threshold=5): - self.access_point = access_point - self.interval = interval - self.target = target - self.healthy_threshold = healthy_threshold - self.timeout = timeout - self.unhealthy_threshold = unhealthy_threshold - - def __repr__(self): - return 'HealthCheck:%s' % self.target - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Interval': - self.interval = int(value) - elif name == 'Target': - self.target = value - elif name == 'HealthyThreshold': - self.healthy_threshold = int(value) - elif name == 'Timeout': - self.timeout = int(value) - elif name == 'UnhealthyThreshold': - self.unhealthy_threshold = int(value) - else: - setattr(self, name, value) - - def update(self): - if not self.access_point: - return - - new_hc = self.connection.configure_health_check(self.access_point, - self) - self.interval = new_hc.interval - self.target = new_hc.target - self.healthy_threshold = new_hc.healthy_threshold - self.unhealthy_threshold = new_hc.unhealthy_threshold - self.timeout = new_hc.timeout - - diff --git a/backup/src/boto/ec2/elb/instancestate.py b/backup/src/boto/ec2/elb/instancestate.py deleted file mode 100644 index 4a9b0d4..0000000 --- a/backup/src/boto/ec2/elb/instancestate.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class InstanceState(object): - """ - Represents the state of an EC2 Load Balancer Instance - """ - - def __init__(self, load_balancer=None, description=None, - state=None, instance_id=None, reason_code=None): - self.load_balancer = load_balancer - self.description = description - self.state = state - self.instance_id = instance_id - self.reason_code = reason_code - - def __repr__(self): - return 'InstanceState:(%s,%s)' % (self.instance_id, self.state) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Description': - self.description = value - elif name == 'State': - self.state = value - elif name == 'InstanceId': - self.instance_id = value - elif name == 'ReasonCode': - self.reason_code = value - else: - setattr(self, name, value) - - - diff --git a/backup/src/boto/ec2/elb/listelement.py b/backup/src/boto/ec2/elb/listelement.py deleted file mode 100644 index 5be4599..0000000 --- a/backup/src/boto/ec2/elb/listelement.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class ListElement(list): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'member': - self.append(value) - - diff --git a/backup/src/boto/ec2/elb/listener.py b/backup/src/boto/ec2/elb/listener.py deleted file mode 100644 index a8807c0..0000000 --- a/backup/src/boto/ec2/elb/listener.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Listener(object): - """ - Represents an EC2 Load Balancer Listener tuple - """ - - def __init__(self, load_balancer=None, load_balancer_port=0, - instance_port=0, protocol='', ssl_certificate_id=None): - self.load_balancer = load_balancer - self.load_balancer_port = load_balancer_port - self.instance_port = instance_port - self.protocol = protocol - self.ssl_certificate_id = ssl_certificate_id - - def __repr__(self): - r = "(%d, %d, '%s'" % (self.load_balancer_port, self.instance_port, self.protocol) - if self.ssl_certificate_id: - r += ', %s' % (self.ssl_certificate_id) - r += ')' - return r - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'LoadBalancerPort': - self.load_balancer_port = int(value) - elif name == 'InstancePort': - self.instance_port = int(value) - elif name == 'Protocol': - self.protocol = value - elif name == 'SSLCertificateId': - self.ssl_certificate_id = value - else: - setattr(self, name, value) - - def get_tuple(self): - return self.load_balancer_port, self.instance_port, self.protocol - - def __getitem__(self, key): - if key == 0: - return self.load_balancer_port - if key == 1: - return self.instance_port - if key == 2: - return self.protocol - raise KeyError - - - - diff --git a/backup/src/boto/ec2/elb/loadbalancer.py b/backup/src/boto/ec2/elb/loadbalancer.py deleted file mode 100644 index 9759952..0000000 --- a/backup/src/boto/ec2/elb/loadbalancer.py +++ /dev/null @@ -1,182 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.ec2.elb.healthcheck import HealthCheck -from boto.ec2.elb.listener import Listener -from boto.ec2.elb.listelement import ListElement -from boto.ec2.elb.policies import Policies -from boto.ec2.instanceinfo import InstanceInfo -from boto.resultset import ResultSet - -class LoadBalancer(object): - """ - Represents an EC2 Load Balancer - """ - - def __init__(self, connection=None, name=None, endpoints=None): - self.connection = connection - self.name = name - self.listeners = None - self.health_check = None - self.policies = None - self.dns_name = None - self.created_time = None - self.instances = None - self.availability_zones = ListElement() - - def __repr__(self): - return 'LoadBalancer:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'HealthCheck': - self.health_check = HealthCheck(self) - return self.health_check - elif name == 'ListenerDescriptions': - self.listeners = ResultSet([('member', Listener)]) - return self.listeners - elif name == 'AvailabilityZones': - return self.availability_zones - elif name == 'Instances': - self.instances = ResultSet([('member', InstanceInfo)]) - return self.instances - elif name == 'Policies': - self.policies = Policies(self) - return self.policies - else: - return None - - def endElement(self, name, value, connection): - if name == 'LoadBalancerName': - self.name = value - elif name == 'DNSName': - self.dns_name = value - elif name == 'CreatedTime': - self.created_time = value - elif name == 'InstanceId': - self.instances.append(value) - else: - setattr(self, name, value) - - def enable_zones(self, zones): - """ - Enable availability zones to this Access Point. - All zones must be in the same region as the Access Point. - - :type zones: string or List of strings - :param zones: The name of the zone(s) to add. - - """ - if isinstance(zones, str) or isinstance(zones, unicode): - zones = [zones] - new_zones = self.connection.enable_availability_zones(self.name, zones) - self.availability_zones = new_zones - - def disable_zones(self, zones): - """ - Disable availability zones from this Access Point. - - :type zones: string or List of strings - :param zones: The name of the zone(s) to add. - - """ - if isinstance(zones, str) or isinstance(zones, unicode): - zones = [zones] - new_zones = self.connection.disable_availability_zones(self.name, zones) - self.availability_zones = new_zones - - def register_instances(self, instances): - """ - Add instances to this Load Balancer - All instances must be in the same region as the Load Balancer. - Adding endpoints that are already registered with the Load Balancer - has no effect. - - :type zones: string or List of instance id's - :param zones: The name of the endpoint(s) to add. - - """ - if isinstance(instances, str) or isinstance(instances, unicode): - instances = [instances] - new_instances = self.connection.register_instances(self.name, instances) - self.instances = new_instances - - def deregister_instances(self, instances): - """ - Remove instances from this Load Balancer. - Removing instances that are not registered with the Load Balancer - has no effect. - - :type zones: string or List of instance id's - :param zones: The name of the endpoint(s) to add. - - """ - if isinstance(instances, str) or isinstance(instances, unicode): - instances = [instances] - new_instances = self.connection.deregister_instances(self.name, instances) - self.instances = new_instances - - def delete(self): - """ - Delete this load balancer - """ - return self.connection.delete_load_balancer(self.name) - - def configure_health_check(self, health_check): - return self.connection.configure_health_check(self.name, health_check) - - def get_instance_health(self, instances=None): - return self.connection.describe_instance_health(self.name, instances) - - def create_listeners(self, listeners): - return self.connection.create_load_balancer_listeners(self.name, listeners) - - def create_listener(self, inPort, outPort=None, proto="tcp"): - if outPort == None: - outPort = inPort - return self.create_listeners([(inPort, outPort, proto)]) - - def delete_listeners(self, listeners): - return self.connection.delete_load_balancer_listeners(self.name, listeners) - - def delete_listener(self, inPort, outPort=None, proto="tcp"): - if outPort == None: - outPort = inPort - return self.delete_listeners([(inPort, outPort, proto)]) - - def delete_policy(self, policy_name): - """ - Deletes a policy from the LoadBalancer. The specified policy must not - be enabled for any listeners. - """ - return self.connection.delete_lb_policy(self.name, policy_name) - - def set_policies_of_listener(self, lb_port, policies): - return self.connection.set_lb_policies_of_listener(self.name, lb_port, policies) - - def create_cookie_stickiness_policy(self, cookie_expiration_period, policy_name): - return self.connection.create_lb_cookie_stickiness_policy(cookie_expiration_period, self.name, policy_name) - - def create_app_cookie_stickiness_policy(self, name, policy_name): - return self.connection.create_app_cookie_stickiness_policy(name, self.name, policy_name) - - def set_listener_SSL_certificate(self, lb_port, ssl_certificate_id): - return self.connection.set_lb_listener_SSL_certificate(self.name, lb_port, ssl_certificate_id) - diff --git a/backup/src/boto/ec2/elb/policies.py b/backup/src/boto/ec2/elb/policies.py deleted file mode 100644 index 428ce72..0000000 --- a/backup/src/boto/ec2/elb/policies.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright (c) 2010 Reza Lotun http://reza.lotun.name -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.resultset import ResultSet - - -class AppCookieStickinessPolicy(object): - def __init__(self, connection=None): - self.cookie_name = None - self.policy_name = None - - def __repr__(self): - return 'AppCookieStickiness(%s, %s)' % (self.policy_name, self.cookie_name) - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'CookieName': - self.cookie_name = value - elif name == 'PolicyName': - self.policy_name = value - - -class LBCookieStickinessPolicy(object): - def __init__(self, connection=None): - self.policy_name = None - self.cookie_expiration_period = None - - def __repr__(self): - return 'LBCookieStickiness(%s, %s)' % (self.policy_name, self.cookie_expiration_period) - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'CookieExpirationPeriod': - self.cookie_expiration_period = value - elif name == 'PolicyName': - self.policy_name = value - - -class Policies(object): - """ - ELB Policies - """ - def __init__(self, connection=None): - self.connection = connection - self.app_cookie_stickiness_policies = None - self.lb_cookie_stickiness_policies = None - - def __repr__(self): - return 'Policies(AppCookieStickiness%s, LBCookieStickiness%s)' % (self.app_cookie_stickiness_policies, - self.lb_cookie_stickiness_policies) - - def startElement(self, name, attrs, connection): - if name == 'AppCookieStickinessPolicies': - self.app_cookie_stickiness_policies = ResultSet([('member', AppCookieStickinessPolicy)]) - elif name == 'LBCookieStickinessPolicies': - self.lb_cookie_stickiness_policies = ResultSet([('member', LBCookieStickinessPolicy)]) - - def endElement(self, name, value, connection): - return - diff --git a/backup/src/boto/ec2/image.py b/backup/src/boto/ec2/image.py deleted file mode 100644 index a85fba0..0000000 --- a/backup/src/boto/ec2/image.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.ec2.ec2object import EC2Object, TaggedEC2Object -from boto.ec2.blockdevicemapping import BlockDeviceMapping - -class ProductCodes(list): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'productCode': - self.append(value) - -class Image(TaggedEC2Object): - """ - Represents an EC2 Image - """ - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.location = None - self.state = None - self.ownerId = None # for backwards compatibility - self.owner_id = None - self.owner_alias = None - self.is_public = False - self.architecture = None - self.platform = None - self.type = None - self.kernel_id = None - self.ramdisk_id = None - self.name = None - self.description = None - self.product_codes = ProductCodes() - self.block_device_mapping = None - self.root_device_type = None - self.root_device_name = None - self.virtualization_type = None - self.hypervisor = None - self.instance_lifecycle = None - - def __repr__(self): - return 'Image:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'blockDeviceMapping': - self.block_device_mapping = BlockDeviceMapping() - return self.block_device_mapping - elif name == 'productCodes': - return self.product_codes - else: - return None - - def endElement(self, name, value, connection): - if name == 'imageId': - self.id = value - elif name == 'imageLocation': - self.location = value - elif name == 'imageState': - self.state = value - elif name == 'imageOwnerId': - self.ownerId = value # for backwards compatibility - self.owner_id = value - elif name == 'isPublic': - if value == 'false': - self.is_public = False - elif value == 'true': - self.is_public = True - else: - raise Exception( - 'Unexpected value of isPublic %s for image %s'%( - value, - self.id - ) - ) - elif name == 'architecture': - self.architecture = value - elif name == 'imageType': - self.type = value - elif name == 'kernelId': - self.kernel_id = value - elif name == 'ramdiskId': - self.ramdisk_id = value - elif name == 'imageOwnerAlias': - self.owner_alias = value - elif name == 'platform': - self.platform = value - elif name == 'name': - self.name = value - elif name == 'description': - self.description = value - elif name == 'rootDeviceType': - self.root_device_type = value - elif name == 'rootDeviceName': - self.root_device_name = value - elif name == 'virtualizationType': - self.virtualization_type = value - elif name == 'hypervisor': - self.hypervisor = value - elif name == 'instanceLifecycle': - self.instance_lifecycle = value - else: - setattr(self, name, value) - - def _update(self, updated): - self.__dict__.update(updated.__dict__) - - def update(self, validate=False): - """ - Update the image's state information by making a call to fetch - the current image attributes from the service. - - :type validate: bool - :param validate: By default, if EC2 returns no data about the - image the update method returns quietly. If - the validate param is True, however, it will - raise a ValueError exception if no data is - returned from EC2. - """ - rs = self.connection.get_all_images([self.id]) - if len(rs) > 0: - img = rs[0] - if img.id == self.id: - self._update(img) - elif validate: - raise ValueError('%s is not a valid Image ID' % self.id) - return self.state - - def run(self, min_count=1, max_count=1, key_name=None, - security_groups=None, user_data=None, - addressing_type=None, instance_type='m1.small', placement=None, - kernel_id=None, ramdisk_id=None, - monitoring_enabled=False, subnet_id=None, - block_device_map=None, - disable_api_termination=False, - instance_initiated_shutdown_behavior=None, - private_ip_address=None, - placement_group=None): - """ - Runs this instance. - - :type min_count: int - :param min_count: The minimum number of instances to start - - :type max_count: int - :param max_count: The maximum number of instances to start - - :type key_name: string - :param key_name: The name of the keypair to run this instance with. - - :type security_groups: - :param security_groups: - - :type user_data: - :param user_data: - - :type addressing_type: - :param daddressing_type: - - :type instance_type: string - :param instance_type: The type of instance to run. Current choices are: - m1.small | m1.large | m1.xlarge | c1.medium | - c1.xlarge | m2.xlarge | m2.2xlarge | - m2.4xlarge | cc1.4xlarge - - :type placement: string - :param placement: The availability zone in which to launch the instances - - :type kernel_id: string - :param kernel_id: The ID of the kernel with which to launch the instances - - :type ramdisk_id: string - :param ramdisk_id: The ID of the RAM disk with which to launch the instances - - :type monitoring_enabled: bool - :param monitoring_enabled: Enable CloudWatch monitoring on the instance. - - :type subnet_id: string - :param subnet_id: The subnet ID within which to launch the instances for VPC. - - :type private_ip_address: string - :param private_ip_address: If you're using VPC, you can optionally use - this parameter to assign the instance a - specific available IP address from the - subnet (e.g., 10.0.0.25). - - :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` - :param block_device_map: A BlockDeviceMapping data structure - describing the EBS volumes associated - with the Image. - - :type disable_api_termination: bool - :param disable_api_termination: If True, the instances will be locked - and will not be able to be terminated - via the API. - - :type instance_initiated_shutdown_behavior: string - :param instance_initiated_shutdown_behavior: Specifies whether the instance's - EBS volumes are stopped (i.e. detached) - or terminated (i.e. deleted) when - the instance is shutdown by the - owner. Valid values are: - stop | terminate - - :type placement_group: string - :param placement_group: If specified, this is the name of the placement - group in which the instance(s) will be launched. - - :rtype: Reservation - :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines - """ - return self.connection.run_instances(self.id, min_count, max_count, - key_name, security_groups, - user_data, addressing_type, - instance_type, placement, - kernel_id, ramdisk_id, - monitoring_enabled, subnet_id, - block_device_map, disable_api_termination, - instance_initiated_shutdown_behavior, - private_ip_address, - placement_group) - - def deregister(self): - return self.connection.deregister_image(self.id) - - def get_launch_permissions(self): - img_attrs = self.connection.get_image_attribute(self.id, - 'launchPermission') - return img_attrs.attrs - - def set_launch_permissions(self, user_ids=None, group_names=None): - return self.connection.modify_image_attribute(self.id, - 'launchPermission', - 'add', - user_ids, - group_names) - - def remove_launch_permissions(self, user_ids=None, group_names=None): - return self.connection.modify_image_attribute(self.id, - 'launchPermission', - 'remove', - user_ids, - group_names) - - def reset_launch_attributes(self): - return self.connection.reset_image_attribute(self.id, - 'launchPermission') - - def get_kernel(self): - img_attrs =self.connection.get_image_attribute(self.id, 'kernel') - return img_attrs.kernel - - def get_ramdisk(self): - img_attrs = self.connection.get_image_attribute(self.id, 'ramdisk') - return img_attrs.ramdisk - -class ImageAttribute: - - def __init__(self, parent=None): - self.name = None - self.kernel = None - self.ramdisk = None - self.attrs = {} - - def startElement(self, name, attrs, connection): - if name == 'blockDeviceMapping': - self.attrs['block_device_mapping'] = BlockDeviceMapping() - return self.attrs['block_device_mapping'] - else: - return None - - def endElement(self, name, value, connection): - if name == 'launchPermission': - self.name = 'launch_permission' - elif name == 'group': - if self.attrs.has_key('groups'): - self.attrs['groups'].append(value) - else: - self.attrs['groups'] = [value] - elif name == 'userId': - if self.attrs.has_key('user_ids'): - self.attrs['user_ids'].append(value) - else: - self.attrs['user_ids'] = [value] - elif name == 'productCode': - if self.attrs.has_key('product_codes'): - self.attrs['product_codes'].append(value) - else: - self.attrs['product_codes'] = [value] - elif name == 'imageId': - self.image_id = value - elif name == 'kernel': - self.kernel = value - elif name == 'ramdisk': - self.ramdisk = value - else: - setattr(self, name, value) diff --git a/backup/src/boto/ec2/instance.py b/backup/src/boto/ec2/instance.py deleted file mode 100644 index 9e8aacf..0000000 --- a/backup/src/boto/ec2/instance.py +++ /dev/null @@ -1,394 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Instance -""" -import boto -from boto.ec2.ec2object import EC2Object, TaggedEC2Object -from boto.resultset import ResultSet -from boto.ec2.address import Address -from boto.ec2.blockdevicemapping import BlockDeviceMapping -from boto.ec2.image import ProductCodes -import base64 - -class Reservation(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.id = None - self.owner_id = None - self.groups = [] - self.instances = [] - - def __repr__(self): - return 'Reservation:%s' % self.id - - def startElement(self, name, attrs, connection): - if name == 'instancesSet': - self.instances = ResultSet([('item', Instance)]) - return self.instances - elif name == 'groupSet': - self.groups = ResultSet([('item', Group)]) - return self.groups - else: - return None - - def endElement(self, name, value, connection): - if name == 'reservationId': - self.id = value - elif name == 'ownerId': - self.owner_id = value - else: - setattr(self, name, value) - - def stop_all(self): - for instance in self.instances: - instance.stop() - -class Instance(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.dns_name = None - self.public_dns_name = None - self.private_dns_name = None - self.state = None - self.state_code = None - self.key_name = None - self.shutdown_state = None - self.previous_state = None - self.instance_type = None - self.instance_class = None - self.launch_time = None - self.image_id = None - self.placement = None - self.kernel = None - self.ramdisk = None - self.product_codes = ProductCodes() - self.ami_launch_index = None - self.monitored = False - self.instance_class = None - self.spot_instance_request_id = None - self.subnet_id = None - self.vpc_id = None - self.private_ip_address = None - self.ip_address = None - self.requester_id = None - self._in_monitoring_element = False - self.persistent = False - self.root_device_name = None - self.root_device_type = None - self.block_device_mapping = None - self.state_reason = None - self.group_name = None - self.client_token = None - - def __repr__(self): - return 'Instance:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'monitoring': - self._in_monitoring_element = True - elif name == 'blockDeviceMapping': - self.block_device_mapping = BlockDeviceMapping() - return self.block_device_mapping - elif name == 'productCodes': - return self.product_codes - elif name == 'stateReason': - self.state_reason = StateReason() - return self.state_reason - return None - - def endElement(self, name, value, connection): - if name == 'instanceId': - self.id = value - elif name == 'imageId': - self.image_id = value - elif name == 'dnsName' or name == 'publicDnsName': - self.dns_name = value # backwards compatibility - self.public_dns_name = value - elif name == 'privateDnsName': - self.private_dns_name = value - elif name == 'keyName': - self.key_name = value - elif name == 'amiLaunchIndex': - self.ami_launch_index = value - elif name == 'shutdownState': - self.shutdown_state = value - elif name == 'previousState': - self.previous_state = value - elif name == 'name': - self.state = value - elif name == 'code': - try: - self.state_code = int(value) - except ValueError: - boto.log.warning('Error converting code (%s) to int' % value) - self.state_code = value - elif name == 'instanceType': - self.instance_type = value - elif name == 'instanceClass': - self.instance_class = value - elif name == 'rootDeviceName': - self.root_device_name = value - elif name == 'rootDeviceType': - self.root_device_type = value - elif name == 'launchTime': - self.launch_time = value - elif name == 'availabilityZone': - self.placement = value - elif name == 'placement': - pass - elif name == 'kernelId': - self.kernel = value - elif name == 'ramdiskId': - self.ramdisk = value - elif name == 'state': - if self._in_monitoring_element: - if value == 'enabled': - self.monitored = True - self._in_monitoring_element = False - elif name == 'instanceClass': - self.instance_class = value - elif name == 'spotInstanceRequestId': - self.spot_instance_request_id = value - elif name == 'subnetId': - self.subnet_id = value - elif name == 'vpcId': - self.vpc_id = value - elif name == 'privateIpAddress': - self.private_ip_address = value - elif name == 'ipAddress': - self.ip_address = value - elif name == 'requesterId': - self.requester_id = value - elif name == 'persistent': - if value == 'true': - self.persistent = True - else: - self.persistent = False - elif name == 'groupName': - if self._in_monitoring_element: - self.group_name = value - elif name == 'clientToken': - self.client_token = value - else: - setattr(self, name, value) - - def _update(self, updated): - self.__dict__.update(updated.__dict__) - - def update(self, validate=False): - """ - Update the instance's state information by making a call to fetch - the current instance attributes from the service. - - :type validate: bool - :param validate: By default, if EC2 returns no data about the - instance the update method returns quietly. If - the validate param is True, however, it will - raise a ValueError exception if no data is - returned from EC2. - """ - rs = self.connection.get_all_instances([self.id]) - if len(rs) > 0: - r = rs[0] - for i in r.instances: - if i.id == self.id: - self._update(i) - elif validate: - raise ValueError('%s is not a valid Instance ID' % self.id) - return self.state - - def terminate(self): - """ - Terminate the instance - """ - rs = self.connection.terminate_instances([self.id]) - self._update(rs[0]) - - def stop(self, force=False): - """ - Stop the instance - - :type force: bool - :param force: Forces the instance to stop - - :rtype: list - :return: A list of the instances stopped - """ - rs = self.connection.stop_instances([self.id]) - self._update(rs[0]) - - def start(self): - """ - Start the instance. - """ - rs = self.connection.start_instances([self.id]) - self._update(rs[0]) - - def reboot(self): - return self.connection.reboot_instances([self.id]) - - def get_console_output(self): - """ - Retrieves the console output for the instance. - - :rtype: :class:`boto.ec2.instance.ConsoleOutput` - :return: The console output as a ConsoleOutput object - """ - return self.connection.get_console_output(self.id) - - def confirm_product(self, product_code): - return self.connection.confirm_product_instance(self.id, product_code) - - def use_ip(self, ip_address): - if isinstance(ip_address, Address): - ip_address = ip_address.public_ip - return self.connection.associate_address(self.id, ip_address) - - def monitor(self): - return self.connection.monitor_instance(self.id) - - def unmonitor(self): - return self.connection.unmonitor_instance(self.id) - - def get_attribute(self, attribute): - """ - Gets an attribute from this instance. - - :type attribute: string - :param attribute: The attribute you need information about - Valid choices are: - instanceType|kernel|ramdisk|userData| - disableApiTermination| - instanceInitiatedShutdownBehavior| - rootDeviceName|blockDeviceMapping - - :rtype: :class:`boto.ec2.image.InstanceAttribute` - :return: An InstanceAttribute object representing the value of the - attribute requested - """ - return self.connection.get_instance_attribute(self.id, attribute) - - def modify_attribute(self, attribute, value): - """ - Changes an attribute of this instance - - :type attribute: string - :param attribute: The attribute you wish to change. - AttributeName - Expected value (default) - instanceType - A valid instance type (m1.small) - kernel - Kernel ID (None) - ramdisk - Ramdisk ID (None) - userData - Base64 encoded String (None) - disableApiTermination - Boolean (true) - instanceInitiatedShutdownBehavior - stop|terminate - rootDeviceName - device name (None) - - :type value: string - :param value: The new value for the attribute - - :rtype: bool - :return: Whether the operation succeeded or not - """ - return self.connection.modify_instance_attribute(self.id, attribute, - value) - - def reset_attribute(self, attribute): - """ - Resets an attribute of this instance to its default value. - - :type attribute: string - :param attribute: The attribute to reset. Valid values are: - kernel|ramdisk - - :rtype: bool - :return: Whether the operation succeeded or not - """ - return self.connection.reset_instance_attribute(self.id, attribute) - -class Group: - - def __init__(self, parent=None): - self.id = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'groupId': - self.id = value - else: - setattr(self, name, value) - -class ConsoleOutput: - - def __init__(self, parent=None): - self.parent = parent - self.instance_id = None - self.timestamp = None - self.comment = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'instanceId': - self.instance_id = value - elif name == 'output': - self.output = base64.b64decode(value) - else: - setattr(self, name, value) - -class InstanceAttribute(dict): - - def __init__(self, parent=None): - dict.__init__(self) - self._current_value = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'value': - self._current_value = value - else: - self[name] = self._current_value - -class StateReason(dict): - - def __init__(self, parent=None): - dict.__init__(self) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name != 'stateReason': - self[name] = value - diff --git a/backup/src/boto/ec2/instanceinfo.py b/backup/src/boto/ec2/instanceinfo.py deleted file mode 100644 index 6efbaed..0000000 --- a/backup/src/boto/ec2/instanceinfo.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class InstanceInfo(object): - """ - Represents an EC2 Instance status response from CloudWatch - """ - - def __init__(self, connection=None, id=None, state=None): - self.connection = connection - self.id = id - self.state = state - - def __repr__(self): - return 'InstanceInfo:%s' % self.id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'instanceId' or name == 'InstanceId': - self.id = value - elif name == 'state': - self.state = value - else: - setattr(self, name, value) - - - diff --git a/backup/src/boto/ec2/keypair.py b/backup/src/boto/ec2/keypair.py deleted file mode 100644 index d08e5ce..0000000 --- a/backup/src/boto/ec2/keypair.py +++ /dev/null @@ -1,111 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Keypair -""" - -import os -from boto.ec2.ec2object import EC2Object -from boto.exception import BotoClientError - -class KeyPair(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.name = None - self.fingerprint = None - self.material = None - - def __repr__(self): - return 'KeyPair:%s' % self.name - - def endElement(self, name, value, connection): - if name == 'keyName': - self.name = value - elif name == 'keyFingerprint': - self.fingerprint = value - elif name == 'keyMaterial': - self.material = value - else: - setattr(self, name, value) - - def delete(self): - """ - Delete the KeyPair. - - :rtype: bool - :return: True if successful, otherwise False. - """ - return self.connection.delete_key_pair(self.name) - - def save(self, directory_path): - """ - Save the material (the unencrypted PEM encoded RSA private key) - of a newly created KeyPair to a local file. - - :type directory_path: string - :param directory_path: The fully qualified path to the directory - in which the keypair will be saved. The - keypair file will be named using the name - of the keypair as the base name and .pem - for the file extension. If a file of that - name already exists in the directory, an - exception will be raised and the old file - will not be overwritten. - - :rtype: bool - :return: True if successful. - """ - if self.material: - file_path = os.path.join(directory_path, '%s.pem' % self.name) - if os.path.exists(file_path): - raise BotoClientError('%s already exists, it will not be overwritten' % file_path) - fp = open(file_path, 'wb') - fp.write(self.material) - fp.close() - return True - else: - raise BotoClientError('KeyPair contains no material') - - def copy_to_region(self, region): - """ - Create a new key pair of the same new in another region. - Note that the new key pair will use a different ssh - cert than the this key pair. After doing the copy, - you will need to save the material associated with the - new key pair (use the save method) to a local file. - - :type region: :class:`boto.ec2.regioninfo.RegionInfo` - :param region: The region to which this security group will be copied. - - :rtype: :class:`boto.ec2.keypair.KeyPair` - :return: The new key pair - """ - if region.name == self.region: - raise BotoClientError('Unable to copy to the same Region') - conn_params = self.connection.get_params() - rconn = region.connect(**conn_params) - kp = rconn.create_key_pair(self.name) - return kp - - - diff --git a/backup/src/boto/ec2/launchspecification.py b/backup/src/boto/ec2/launchspecification.py deleted file mode 100644 index a574a38..0000000 --- a/backup/src/boto/ec2/launchspecification.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a launch specification for Spot instances. -""" - -from boto.ec2.ec2object import EC2Object -from boto.resultset import ResultSet -from boto.ec2.blockdevicemapping import BlockDeviceMapping -from boto.ec2.instance import Group - -class GroupList(list): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'groupId': - self.append(value) - -class LaunchSpecification(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.key_name = None - self.instance_type = None - self.image_id = None - self.groups = [] - self.placement = None - self.kernel = None - self.ramdisk = None - self.monitored = False - self.subnet_id = None - self._in_monitoring_element = False - self.block_device_mapping = None - - def __repr__(self): - return 'LaunchSpecification(%s)' % self.image_id - - def startElement(self, name, attrs, connection): - if name == 'groupSet': - self.groups = ResultSet([('item', Group)]) - return self.groups - elif name == 'monitoring': - self._in_monitoring_element = True - elif name == 'blockDeviceMapping': - self.block_device_mapping = BlockDeviceMapping() - return self.block_device_mapping - else: - return None - - def endElement(self, name, value, connection): - if name == 'imageId': - self.image_id = value - elif name == 'keyName': - self.key_name = value - elif name == 'instanceType': - self.instance_type = value - elif name == 'availabilityZone': - self.placement = value - elif name == 'placement': - pass - elif name == 'kernelId': - self.kernel = value - elif name == 'ramdiskId': - self.ramdisk = value - elif name == 'subnetId': - self.subnet_id = value - elif name == 'state': - if self._in_monitoring_element: - if value == 'enabled': - self.monitored = True - self._in_monitoring_element = False - else: - setattr(self, name, value) - - diff --git a/backup/src/boto/ec2/placementgroup.py b/backup/src/boto/ec2/placementgroup.py deleted file mode 100644 index e1bbea6..0000000 --- a/backup/src/boto/ec2/placementgroup.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -""" -Represents an EC2 Placement Group -""" -from boto.ec2.ec2object import EC2Object -from boto.exception import BotoClientError - -class PlacementGroup(EC2Object): - - def __init__(self, connection=None, name=None, strategy=None, state=None): - EC2Object.__init__(self, connection) - self.name = name - self.strategy = strategy - self.state = state - - def __repr__(self): - return 'PlacementGroup:%s' % self.name - - def endElement(self, name, value, connection): - if name == 'groupName': - self.name = value - elif name == 'strategy': - self.strategy = value - elif name == 'state': - self.state = value - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_placement_group(self.name) - - diff --git a/backup/src/boto/ec2/regioninfo.py b/backup/src/boto/ec2/regioninfo.py deleted file mode 100644 index 0b37b0e..0000000 --- a/backup/src/boto/ec2/regioninfo.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.regioninfo import RegionInfo - -class EC2RegionInfo(RegionInfo): - """ - Represents an EC2 Region - """ - - def __init__(self, connection=None, name=None, endpoint=None): - from boto.ec2.connection import EC2Connection - RegionInfo.__init__(self, connection, name, endpoint, - EC2Connection) diff --git a/backup/src/boto/ec2/reservedinstance.py b/backup/src/boto/ec2/reservedinstance.py deleted file mode 100644 index 1d35c1d..0000000 --- a/backup/src/boto/ec2/reservedinstance.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.ec2.ec2object import EC2Object - -class ReservedInstancesOffering(EC2Object): - - def __init__(self, connection=None, id=None, instance_type=None, - availability_zone=None, duration=None, fixed_price=None, - usage_price=None, description=None): - EC2Object.__init__(self, connection) - self.id = id - self.instance_type = instance_type - self.availability_zone = availability_zone - self.duration = duration - self.fixed_price = fixed_price - self.usage_price = usage_price - self.description = description - - def __repr__(self): - return 'ReservedInstanceOffering:%s' % self.id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'reservedInstancesOfferingId': - self.id = value - elif name == 'instanceType': - self.instance_type = value - elif name == 'availabilityZone': - self.availability_zone = value - elif name == 'duration': - self.duration = value - elif name == 'fixedPrice': - self.fixed_price = value - elif name == 'usagePrice': - self.usage_price = value - elif name == 'productDescription': - self.description = value - else: - setattr(self, name, value) - - def describe(self): - print 'ID=%s' % self.id - print '\tInstance Type=%s' % self.instance_type - print '\tZone=%s' % self.availability_zone - print '\tDuration=%s' % self.duration - print '\tFixed Price=%s' % self.fixed_price - print '\tUsage Price=%s' % self.usage_price - print '\tDescription=%s' % self.description - - def purchase(self, instance_count=1): - return self.connection.purchase_reserved_instance_offering(self.id, instance_count) - -class ReservedInstance(ReservedInstancesOffering): - - def __init__(self, connection=None, id=None, instance_type=None, - availability_zone=None, duration=None, fixed_price=None, - usage_price=None, description=None, - instance_count=None, state=None): - ReservedInstancesOffering.__init__(self, connection, id, instance_type, - availability_zone, duration, fixed_price, - usage_price, description) - self.instance_count = instance_count - self.state = state - - def __repr__(self): - return 'ReservedInstance:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'reservedInstancesId': - self.id = value - if name == 'instanceCount': - self.instance_count = int(value) - elif name == 'state': - self.state = value - else: - ReservedInstancesOffering.endElement(self, name, value, connection) diff --git a/backup/src/boto/ec2/securitygroup.py b/backup/src/boto/ec2/securitygroup.py deleted file mode 100644 index 24e08c3..0000000 --- a/backup/src/boto/ec2/securitygroup.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Security Group -""" -from boto.ec2.ec2object import EC2Object -from boto.exception import BotoClientError - -class SecurityGroup(EC2Object): - - def __init__(self, connection=None, owner_id=None, - name=None, description=None): - EC2Object.__init__(self, connection) - self.owner_id = owner_id - self.name = name - self.description = description - self.rules = [] - - def __repr__(self): - return 'SecurityGroup:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'item': - self.rules.append(IPPermissions(self)) - return self.rules[-1] - else: - return None - - def endElement(self, name, value, connection): - if name == 'ownerId': - self.owner_id = value - elif name == 'groupName': - self.name = value - elif name == 'groupDescription': - self.description = value - elif name == 'ipRanges': - pass - elif name == 'return': - if value == 'false': - self.status = False - elif value == 'true': - self.status = True - else: - raise Exception( - 'Unexpected value of status %s for group %s'%( - value, - self.name - ) - ) - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_security_group(self.name) - - def add_rule(self, ip_protocol, from_port, to_port, - src_group_name, src_group_owner_id, cidr_ip): - """ - Add a rule to the SecurityGroup object. Note that this method - only changes the local version of the object. No information - is sent to EC2. - """ - rule = IPPermissions(self) - rule.ip_protocol = ip_protocol - rule.from_port = from_port - rule.to_port = to_port - self.rules.append(rule) - rule.add_grant(src_group_name, src_group_owner_id, cidr_ip) - - def remove_rule(self, ip_protocol, from_port, to_port, - src_group_name, src_group_owner_id, cidr_ip): - """ - Remove a rule to the SecurityGroup object. Note that this method - only changes the local version of the object. No information - is sent to EC2. - """ - target_rule = None - for rule in self.rules: - if rule.ip_protocol == ip_protocol: - if rule.from_port == from_port: - if rule.to_port == to_port: - target_rule = rule - target_grant = None - for grant in rule.grants: - if grant.name == src_group_name: - if grant.owner_id == src_group_owner_id: - if grant.cidr_ip == cidr_ip: - target_grant = grant - if target_grant: - rule.grants.remove(target_grant) - if len(rule.grants) == 0: - self.rules.remove(target_rule) - - def authorize(self, ip_protocol=None, from_port=None, to_port=None, - cidr_ip=None, src_group=None): - """ - Add a new rule to this security group. - You need to pass in either src_group_name - OR ip_protocol, from_port, to_port, - and cidr_ip. In other words, either you are authorizing another - group or you are authorizing some ip-based rule. - - :type ip_protocol: string - :param ip_protocol: Either tcp | udp | icmp - - :type from_port: int - :param from_port: The beginning port number you are enabling - - :type to_port: int - :param to_port: The ending port number you are enabling - - :type to_port: string - :param to_port: The CIDR block you are providing access to. - See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - - :type src_group: :class:`boto.ec2.securitygroup.SecurityGroup` or - :class:`boto.ec2.securitygroup.GroupOrCIDR` - - :rtype: bool - :return: True if successful. - """ - if src_group: - cidr_ip = None - src_group_name = src_group.name - src_group_owner_id = src_group.owner_id - else: - src_group_name = None - src_group_owner_id = None - status = self.connection.authorize_security_group(self.name, - src_group_name, - src_group_owner_id, - ip_protocol, - from_port, - to_port, - cidr_ip) - if status: - self.add_rule(ip_protocol, from_port, to_port, src_group_name, - src_group_owner_id, cidr_ip) - return status - - def revoke(self, ip_protocol=None, from_port=None, to_port=None, - cidr_ip=None, src_group=None): - if src_group: - cidr_ip=None - src_group_name = src_group.name - src_group_owner_id = src_group.owner_id - else: - src_group_name = None - src_group_owner_id = None - status = self.connection.revoke_security_group(self.name, - src_group_name, - src_group_owner_id, - ip_protocol, - from_port, - to_port, - cidr_ip) - if status: - self.remove_rule(ip_protocol, from_port, to_port, src_group_name, - src_group_owner_id, cidr_ip) - return status - - def copy_to_region(self, region, name=None): - """ - Create a copy of this security group in another region. - Note that the new security group will be a separate entity - and will not stay in sync automatically after the copy - operation. - - :type region: :class:`boto.ec2.regioninfo.RegionInfo` - :param region: The region to which this security group will be copied. - - :type name: string - :param name: The name of the copy. If not supplied, the copy - will have the same name as this security group. - - :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` - :return: The new security group. - """ - if region.name == self.region: - raise BotoClientError('Unable to copy to the same Region') - conn_params = self.connection.get_params() - rconn = region.connect(**conn_params) - sg = rconn.create_security_group(name or self.name, self.description) - source_groups = [] - for rule in self.rules: - grant = rule.grants[0] - if grant.name: - if grant.name not in source_groups: - source_groups.append(grant.name) - sg.authorize(None, None, None, None, grant) - else: - sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, - grant.cidr_ip) - return sg - - def instances(self): - instances = [] - rs = self.connection.get_all_instances() - for reservation in rs: - uses_group = [g.id for g in reservation.groups if g.id == self.name] - if uses_group: - instances.extend(reservation.instances) - return instances - -class IPPermissions: - - def __init__(self, parent=None): - self.parent = parent - self.ip_protocol = None - self.from_port = None - self.to_port = None - self.grants = [] - - def __repr__(self): - return 'IPPermissions:%s(%s-%s)' % (self.ip_protocol, - self.from_port, self.to_port) - - def startElement(self, name, attrs, connection): - if name == 'item': - self.grants.append(GroupOrCIDR(self)) - return self.grants[-1] - return None - - def endElement(self, name, value, connection): - if name == 'ipProtocol': - self.ip_protocol = value - elif name == 'fromPort': - self.from_port = value - elif name == 'toPort': - self.to_port = value - else: - setattr(self, name, value) - - def add_grant(self, name=None, owner_id=None, cidr_ip=None): - grant = GroupOrCIDR(self) - grant.owner_id = owner_id - grant.name = name - grant.cidr_ip = cidr_ip - self.grants.append(grant) - return grant - -class GroupOrCIDR: - - def __init__(self, parent=None): - self.owner_id = None - self.name = None - self.cidr_ip = None - - def __repr__(self): - if self.cidr_ip: - return '%s' % self.cidr_ip - else: - return '%s-%s' % (self.name, self.owner_id) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'userId': - self.owner_id = value - elif name == 'groupName': - self.name = value - if name == 'cidrIp': - self.cidr_ip = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/ec2/snapshot.py b/backup/src/boto/ec2/snapshot.py deleted file mode 100644 index bbe8ad4..0000000 --- a/backup/src/boto/ec2/snapshot.py +++ /dev/null @@ -1,140 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Elastic IP Snapshot -""" -from boto.ec2.ec2object import TaggedEC2Object - -class Snapshot(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.volume_id = None - self.status = None - self.progress = None - self.start_time = None - self.owner_id = None - self.volume_size = None - self.description = None - - def __repr__(self): - return 'Snapshot:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'snapshotId': - self.id = value - elif name == 'volumeId': - self.volume_id = value - elif name == 'status': - self.status = value - elif name == 'startTime': - self.start_time = value - elif name == 'ownerId': - self.owner_id = value - elif name == 'volumeSize': - try: - self.volume_size = int(value) - except: - self.volume_size = value - elif name == 'description': - self.description = value - else: - setattr(self, name, value) - - def _update(self, updated): - self.progress = updated.progress - self.status = updated.status - - def update(self, validate=False): - """ - Update the data associated with this snapshot by querying EC2. - - :type validate: bool - :param validate: By default, if EC2 returns no data about the - snapshot the update method returns quietly. If - the validate param is True, however, it will - raise a ValueError exception if no data is - returned from EC2. - """ - rs = self.connection.get_all_snapshots([self.id]) - if len(rs) > 0: - self._update(rs[0]) - elif validate: - raise ValueError('%s is not a valid Snapshot ID' % self.id) - return self.progress - - def delete(self): - return self.connection.delete_snapshot(self.id) - - def get_permissions(self): - attrs = self.connection.get_snapshot_attribute(self.id, - attribute='createVolumePermission') - return attrs.attrs - - def share(self, user_ids=None, groups=None): - return self.connection.modify_snapshot_attribute(self.id, - 'createVolumePermission', - 'add', - user_ids, - groups) - - def unshare(self, user_ids=None, groups=None): - return self.connection.modify_snapshot_attribute(self.id, - 'createVolumePermission', - 'remove', - user_ids, - groups) - - def reset_permissions(self): - return self.connection.reset_snapshot_attribute(self.id, 'createVolumePermission') - -class SnapshotAttribute: - - def __init__(self, parent=None): - self.snapshot_id = None - self.attrs = {} - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'createVolumePermission': - self.name = 'create_volume_permission' - elif name == 'group': - if self.attrs.has_key('groups'): - self.attrs['groups'].append(value) - else: - self.attrs['groups'] = [value] - elif name == 'userId': - if self.attrs.has_key('user_ids'): - self.attrs['user_ids'].append(value) - else: - self.attrs['user_ids'] = [value] - elif name == 'snapshotId': - self.snapshot_id = value - else: - setattr(self, name, value) - - - diff --git a/backup/src/boto/ec2/spotdatafeedsubscription.py b/backup/src/boto/ec2/spotdatafeedsubscription.py deleted file mode 100644 index 9b820a3..0000000 --- a/backup/src/boto/ec2/spotdatafeedsubscription.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Spot Instance Datafeed Subscription -""" -from boto.ec2.ec2object import EC2Object -from boto.ec2.spotinstancerequest import SpotInstanceStateFault - -class SpotDatafeedSubscription(EC2Object): - - def __init__(self, connection=None, owner_id=None, - bucket=None, prefix=None, state=None,fault=None): - EC2Object.__init__(self, connection) - self.owner_id = owner_id - self.bucket = bucket - self.prefix = prefix - self.state = state - self.fault = fault - - def __repr__(self): - return 'SpotDatafeedSubscription:%s' % self.bucket - - def startElement(self, name, attrs, connection): - if name == 'fault': - self.fault = SpotInstanceStateFault() - return self.fault - else: - return None - - def endElement(self, name, value, connection): - if name == 'ownerId': - self.owner_id = value - elif name == 'bucket': - self.bucket = value - elif name == 'prefix': - self.prefix = value - elif name == 'state': - self.state = value - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_spot_datafeed_subscription() - diff --git a/backup/src/boto/ec2/spotinstancerequest.py b/backup/src/boto/ec2/spotinstancerequest.py deleted file mode 100644 index 06acb0f..0000000 --- a/backup/src/boto/ec2/spotinstancerequest.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Spot Instance Request -""" - -from boto.ec2.ec2object import TaggedEC2Object -from boto.ec2.launchspecification import LaunchSpecification - -class SpotInstanceStateFault(object): - - def __init__(self, code=None, message=None): - self.code = code - self.message = message - - def __repr__(self): - return '(%s, %s)' % (self.code, self.message) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'code': - self.code = value - elif name == 'message': - self.message = value - setattr(self, name, value) - -class SpotInstanceRequest(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.price = None - self.type = None - self.state = None - self.fault = None - self.valid_from = None - self.valid_until = None - self.launch_group = None - self.product_description = None - self.availability_zone_group = None - self.create_time = None - self.launch_specification = None - self.instance_id = None - - def __repr__(self): - return 'SpotInstanceRequest:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'launchSpecification': - self.launch_specification = LaunchSpecification(connection) - return self.launch_specification - elif name == 'fault': - self.fault = SpotInstanceStateFault() - return self.fault - else: - return None - - def endElement(self, name, value, connection): - if name == 'spotInstanceRequestId': - self.id = value - elif name == 'spotPrice': - self.price = float(value) - elif name == 'type': - self.type = value - elif name == 'state': - self.state = value - elif name == 'productDescription': - self.product_description = value - elif name == 'validFrom': - self.valid_from = value - elif name == 'validUntil': - self.valid_until = value - elif name == 'launchGroup': - self.launch_group = value - elif name == 'availabilityZoneGroup': - self.availability_zone_group = value - elif name == 'createTime': - self.create_time = value - elif name == 'instanceId': - self.instance_id = value - else: - setattr(self, name, value) - - def cancel(self): - self.connection.cancel_spot_instance_requests([self.id]) - - - diff --git a/backup/src/boto/ec2/spotpricehistory.py b/backup/src/boto/ec2/spotpricehistory.py deleted file mode 100644 index d4e1711..0000000 --- a/backup/src/boto/ec2/spotpricehistory.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Spot Instance Request -""" - -from boto.ec2.ec2object import EC2Object - -class SpotPriceHistory(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.price = 0.0 - self.instance_type = None - self.product_description = None - self.timestamp = None - - def __repr__(self): - return 'SpotPriceHistory(%s):%2f' % (self.instance_type, self.price) - - def endElement(self, name, value, connection): - if name == 'instanceType': - self.instance_type = value - elif name == 'spotPrice': - self.price = float(value) - elif name == 'productDescription': - self.product_description = value - elif name == 'timestamp': - self.timestamp = value - else: - setattr(self, name, value) - - diff --git a/backup/src/boto/ec2/tag.py b/backup/src/boto/ec2/tag.py deleted file mode 100644 index 8032e6f..0000000 --- a/backup/src/boto/ec2/tag.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class TagSet(dict): - """ - A TagSet is used to collect the tags associated with a particular - EC2 resource. Not all resources can be tagged but for those that - can, this dict object will be used to collect those values. See - :class:`boto.ec2.ec2object.TaggedEC2Object` for more details. - """ - - def __init__(self, connection=None): - self.connection = connection - self._current_key = None - self._current_value = None - - def startElement(self, name, attrs, connection): - if name == 'item': - self._current_key = None - self._current_value = None - return None - - def endElement(self, name, value, connection): - if name == 'key': - self._current_key = value - elif name == 'value': - self._current_value = value - elif name == 'item': - self[self._current_key] = self._current_value - - -class Tag(object): - """ - A Tag is used when creating or listing all tags related to - an AWS account. It records not only the key and value but - also the ID of the resource to which the tag is attached - as well as the type of the resource. - """ - - def __init__(self, connection=None, res_id=None, res_type=None, - name=None, value=None): - self.connection = connection - self.res_id = res_id - self.res_type = res_type - self.name = name - self.value = value - - def __repr__(self): - return 'Tag:%s' % self.name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'resourceId': - self.res_id = value - elif name == 'resourceType': - self.res_type = value - elif name == 'key': - self.name = value - elif name == 'value': - self.value = value - else: - setattr(self, name, value) - - - - diff --git a/backup/src/boto/ec2/volume.py b/backup/src/boto/ec2/volume.py deleted file mode 100644 index 45345fa..0000000 --- a/backup/src/boto/ec2/volume.py +++ /dev/null @@ -1,227 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Elastic Block Storage Volume -""" -from boto.ec2.ec2object import TaggedEC2Object - -class Volume(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.create_time = None - self.status = None - self.size = None - self.snapshot_id = None - self.attach_data = None - self.zone = None - - def __repr__(self): - return 'Volume:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'attachmentSet': - self.attach_data = AttachmentSet() - return self.attach_data - elif name == 'tagSet': - self.tags = boto.resultset.ResultSet([('item', Tag)]) - return self.tags - else: - return None - - def endElement(self, name, value, connection): - if name == 'volumeId': - self.id = value - elif name == 'createTime': - self.create_time = value - elif name == 'status': - if value != '': - self.status = value - elif name == 'size': - self.size = int(value) - elif name == 'snapshotId': - self.snapshot_id = value - elif name == 'availabilityZone': - self.zone = value - else: - setattr(self, name, value) - - def _update(self, updated): - self.__dict__.update(updated.__dict__) - - def update(self, validate=False): - """ - Update the data associated with this volume by querying EC2. - - :type validate: bool - :param validate: By default, if EC2 returns no data about the - volume the update method returns quietly. If - the validate param is True, however, it will - raise a ValueError exception if no data is - returned from EC2. - """ - rs = self.connection.get_all_volumes([self.id]) - if len(rs) > 0: - self._update(rs[0]) - elif validate: - raise ValueError('%s is not a valid Volume ID' % self.id) - return self.status - - def delete(self): - """ - Delete this EBS volume. - - :rtype: bool - :return: True if successful - """ - return self.connection.delete_volume(self.id) - - def attach(self, instance_id, device): - """ - Attach this EBS volume to an EC2 instance. - - :type instance_id: str - :param instance_id: The ID of the EC2 instance to which it will - be attached. - - :type device: str - :param device: The device on the instance through which the - volume will be exposted (e.g. /dev/sdh) - - :rtype: bool - :return: True if successful - """ - return self.connection.attach_volume(self.id, instance_id, device) - - def detach(self, force=False): - """ - Detach this EBS volume from an EC2 instance. - - :type force: bool - :param force: Forces detachment if the previous detachment attempt did - not occur cleanly. This option can lead to data loss or - a corrupted file system. Use this option only as a last - resort to detach a volume from a failed instance. The - instance will not have an opportunity to flush file system - caches nor file system meta data. If you use this option, - you must perform file system check and repair procedures. - - :rtype: bool - :return: True if successful - """ - instance_id = None - if self.attach_data: - instance_id = self.attach_data.instance_id - device = None - if self.attach_data: - device = self.attach_data.device - return self.connection.detach_volume(self.id, instance_id, device, force) - - def create_snapshot(self, description=None): - """ - Create a snapshot of this EBS Volume. - - :type description: str - :param description: A description of the snapshot. Limited to 256 characters. - - :rtype: bool - :return: True if successful - """ - return self.connection.create_snapshot(self.id, description) - - def volume_state(self): - """ - Returns the state of the volume. Same value as the status attribute. - """ - return self.status - - def attachment_state(self): - """ - Get the attachment state. - """ - state = None - if self.attach_data: - state = self.attach_data.status - return state - - def snapshots(self, owner=None, restorable_by=None): - """ - Get all snapshots related to this volume. Note that this requires - that all available snapshots for the account be retrieved from EC2 - first and then the list is filtered client-side to contain only - those for this volume. - - :type owner: str - :param owner: If present, only the snapshots owned by the specified user - will be returned. Valid values are: - self | amazon | AWS Account ID - - :type restorable_by: str - :param restorable_by: If present, only the snapshots that are restorable - by the specified account id will be returned. - - :rtype: list of L{boto.ec2.snapshot.Snapshot} - :return: The requested Snapshot objects - - """ - rs = self.connection.get_all_snapshots(owner=owner, - restorable_by=restorable_by) - mine = [] - for snap in rs: - if snap.volume_id == self.id: - mine.append(snap) - return mine - -class AttachmentSet(object): - - def __init__(self): - self.id = None - self.instance_id = None - self.status = None - self.attach_time = None - self.device = None - - def __repr__(self): - return 'AttachmentSet:%s' % self.id - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'volumeId': - self.id = value - elif name == 'instanceId': - self.instance_id = value - elif name == 'status': - self.status = value - elif name == 'attachTime': - self.attach_time = value - elif name == 'device': - self.device = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/ec2/zone.py b/backup/src/boto/ec2/zone.py deleted file mode 100644 index aec79b2..0000000 --- a/backup/src/boto/ec2/zone.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an EC2 Availability Zone -""" -from boto.ec2.ec2object import EC2Object - -class Zone(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.name = None - self.state = None - - def __repr__(self): - return 'Zone:%s' % self.name - - def endElement(self, name, value, connection): - if name == 'zoneName': - self.name = value - elif name == 'zoneState': - self.state = value - else: - setattr(self, name, value) - - - - diff --git a/backup/src/boto/ecs/__init__.py b/backup/src/boto/ecs/__init__.py deleted file mode 100644 index db86dd5..0000000 --- a/backup/src/boto/ecs/__init__.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto.connection import AWSQueryConnection, AWSAuthConnection -import time -import urllib -import xml.sax -from boto.ecs.item import ItemSet -from boto import handler - -class ECSConnection(AWSQueryConnection): - """ECommerse Connection""" - - APIVersion = '2010-11-01' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, host='ecs.amazonaws.com', - debug=0, https_connection_factory=None, path='/'): - AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - host, debug, https_connection_factory, path) - - def _required_auth_capability(self): - return ['ecs'] - - def get_response(self, action, params, page=0, itemSet=None): - """ - Utility method to handle calls to ECS and parsing of responses. - """ - params['Service'] = "AWSECommerceService" - params['Operation'] = action - if page: - params['ItemPage'] = page - response = self.make_request(None, params, "/onca/xml") - body = response.read() - boto.log.debug(body) - - if response.status != 200: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - if itemSet == None: - rs = ItemSet(self, action, params, page) - else: - rs = itemSet - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - - # - # Group methods - # - - def item_search(self, search_index, **params): - """ - Returns items that satisfy the search criteria, including one or more search - indices. - - For a full list of search terms, - :see: http://docs.amazonwebservices.com/AWSECommerceService/2010-09-01/DG/index.html?ItemSearch.html - """ - params['SearchIndex'] = search_index - return self.get_response('ItemSearch', params) diff --git a/backup/src/boto/ecs/item.py b/backup/src/boto/ecs/item.py deleted file mode 100644 index 29588b8..0000000 --- a/backup/src/boto/ecs/item.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -import xml.sax -import cgi -from StringIO import StringIO - -class ResponseGroup(xml.sax.ContentHandler): - """A Generic "Response Group", which can - be anything from the entire list of Items to - specific response elements within an item""" - - def __init__(self, connection=None, nodename=None): - """Initialize this Item""" - self._connection = connection - self._nodename = nodename - self._nodepath = [] - self._curobj = None - self._xml = StringIO() - - def __repr__(self): - return '<%s: %s>' % (self.__class__.__name__, self.__dict__) - - # - # Attribute Functions - # - def get(self, name): - return self.__dict__.get(name) - - def set(self, name, value): - self.__dict__[name] = value - - def to_xml(self): - return "<%s>%s" % (self._nodename, self._xml.getvalue(), self._nodename) - - # - # XML Parser functions - # - def startElement(self, name, attrs, connection): - self._xml.write("<%s>" % name) - self._nodepath.append(name) - if len(self._nodepath) == 1: - obj = ResponseGroup(self._connection) - self.set(name, obj) - self._curobj = obj - elif self._curobj: - self._curobj.startElement(name, attrs, connection) - return None - - def endElement(self, name, value, connection): - self._xml.write("%s" % (cgi.escape(value).replace("&amp;", "&"), name)) - if len(self._nodepath) == 0: - return - obj = None - curval = self.get(name) - if len(self._nodepath) == 1: - if value or not curval: - self.set(name, value) - if self._curobj: - self._curobj = None - #elif len(self._nodepath) == 2: - #self._curobj = None - elif self._curobj: - self._curobj.endElement(name, value, connection) - self._nodepath.pop() - return None - - -class Item(ResponseGroup): - """A single Item""" - - def __init__(self, connection=None): - """Initialize this Item""" - ResponseGroup.__init__(self, connection, "Item") - -class ItemSet(ResponseGroup): - """A special ResponseGroup that has built-in paging, and - only creates new Items on the "Item" tag""" - - def __init__(self, connection, action, params, page=0): - ResponseGroup.__init__(self, connection, "Items") - self.objs = [] - self.iter = None - self.page = page - self.action = action - self.params = params - self.curItem = None - self.total_results = 0 - self.total_pages = 0 - - def startElement(self, name, attrs, connection): - if name == "Item": - self.curItem = Item(self._connection) - elif self.curItem != None: - self.curItem.startElement(name, attrs, connection) - return None - - def endElement(self, name, value, connection): - if name == 'TotalResults': - self.total_results = value - elif name == 'TotalPages': - self.total_pages = value - elif name == "Item": - self.objs.append(self.curItem) - self._xml.write(self.curItem.to_xml()) - self.curItem = None - elif self.curItem != None: - self.curItem.endElement(name, value, connection) - return None - - def next(self): - """Special paging functionality""" - if self.iter == None: - self.iter = iter(self.objs) - try: - return self.iter.next() - except StopIteration: - self.iter = None - self.objs = [] - if int(self.page) < int(self.total_pages): - self.page += 1 - self._connection.get_response(self.action, self.params, self.page, self) - return self.next() - else: - raise - - def __iter__(self): - return self - - def to_xml(self): - """Override to first fetch everything""" - for item in self: - pass - return ResponseGroup.to_xml(self) diff --git a/backup/src/boto/emr/__init__.py b/backup/src/boto/emr/__init__.py deleted file mode 100644 index 3c33f9a..0000000 --- a/backup/src/boto/emr/__init__.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2010 Spotify AB -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -This module provies an interface to the Elastic MapReduce (EMR) -service from AWS. -""" -from connection import EmrConnection -from step import Step, StreamingStep, JarStep -from bootstrap_action import BootstrapAction - - diff --git a/backup/src/boto/emr/bootstrap_action.py b/backup/src/boto/emr/bootstrap_action.py deleted file mode 100644 index c1c9038..0000000 --- a/backup/src/boto/emr/bootstrap_action.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2010 Spotify AB -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class BootstrapAction(object): - def __init__(self, name, path, bootstrap_action_args): - self.name = name - self.path = path - - if isinstance(bootstrap_action_args, basestring): - bootstrap_action_args = [bootstrap_action_args] - - self.bootstrap_action_args = bootstrap_action_args - - def args(self): - args = [] - - if self.bootstrap_action_args: - args.extend(self.bootstrap_action_args) - - return args - - def __repr__(self): - return '%s.%s(name=%r, path=%r, bootstrap_action_args=%r)' % ( - self.__class__.__module__, self.__class__.__name__, - self.name, self.path, self.bootstrap_action_args) diff --git a/backup/src/boto/emr/connection.py b/backup/src/boto/emr/connection.py deleted file mode 100644 index f0145e3..0000000 --- a/backup/src/boto/emr/connection.py +++ /dev/null @@ -1,280 +0,0 @@ -# Copyright (c) 2010 Spotify AB -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a connection to the EMR service -""" -import types - -import boto -from boto.ec2.regioninfo import RegionInfo -from boto.emr.emrobject import JobFlow, RunJobFlowResponse -from boto.emr.step import JarStep -from boto.connection import AWSQueryConnection -from boto.exception import EmrResponseError - -class EmrConnection(AWSQueryConnection): - - APIVersion = boto.config.get('Boto', 'emr_version', '2009-03-31') - DefaultRegionName = boto.config.get('Boto', 'emr_region_name', 'us-east-1') - DefaultRegionEndpoint = boto.config.get('Boto', 'emr_region_endpoint', - 'elasticmapreduce.amazonaws.com') - ResponseError = EmrResponseError - - # Constants for AWS Console debugging - DebuggingJar = 's3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar' - DebuggingArgs = 's3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - if not region: - region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['emr'] - - def describe_jobflow(self, jobflow_id): - """ - Describes a single Elastic MapReduce job flow - - :type jobflow_id: str - :param jobflow_id: The job flow id of interest - """ - jobflows = self.describe_jobflows(jobflow_ids=[jobflow_id]) - if jobflows: - return jobflows[0] - - def describe_jobflows(self, states=None, jobflow_ids=None, - created_after=None, created_before=None): - """ - Retrieve all the Elastic MapReduce job flows on your account - - :type states: list - :param states: A list of strings with job flow states wanted - - :type jobflow_ids: list - :param jobflow_ids: A list of job flow IDs - :type created_after: datetime - :param created_after: Bound on job flow creation time - - :type created_before: datetime - :param created_before: Bound on job flow creation time - """ - params = {} - - if states: - self.build_list_params(params, states, 'JobFlowStates.member') - if jobflow_ids: - self.build_list_params(params, jobflow_ids, 'JobFlowIds.member') - if created_after: - params['CreatedAfter'] = created_after.strftime('%Y-%m-%dT%H:%M:%S') - if created_before: - params['CreatedBefore'] = created_before.strftime('%Y-%m-%dT%H:%M:%S') - - return self.get_list('DescribeJobFlows', params, [('member', JobFlow)]) - - def terminate_jobflow(self, jobflow_id): - """ - Terminate an Elastic MapReduce job flow - - :type jobflow_id: str - :param jobflow_id: A jobflow id - """ - self.terminate_jobflows([jobflow_id]) - - def terminate_jobflows(self, jobflow_ids): - """ - Terminate an Elastic MapReduce job flow - - :type jobflow_ids: list - :param jobflow_ids: A list of job flow IDs - """ - params = {} - self.build_list_params(params, jobflow_ids, 'JobFlowIds.member') - return self.get_status('TerminateJobFlows', params) - - def add_jobflow_steps(self, jobflow_id, steps): - """ - Adds steps to a jobflow - - :type jobflow_id: str - :param jobflow_id: The job flow id - :type steps: list(boto.emr.Step) - :param steps: A list of steps to add to the job - """ - if type(steps) != types.ListType: - steps = [steps] - params = {} - params['JobFlowId'] = jobflow_id - - # Step args - step_args = [self._build_step_args(step) for step in steps] - params.update(self._build_step_list(step_args)) - - return self.get_object('AddJobFlowSteps', params, RunJobFlowResponse) - - def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, - master_instance_type='m1.small', - slave_instance_type='m1.small', num_instances=1, - action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False, - enable_debugging=False, - hadoop_version='0.18', - steps=[], - bootstrap_actions=[]): - """ - Runs a job flow - - :type name: str - :param name: Name of the job flow - :type log_uri: str - :param log_uri: URI of the S3 bucket to place logs - :type ec2_keyname: str - :param ec2_keyname: EC2 key used for the instances - :type availability_zone: str - :param availability_zone: EC2 availability zone of the cluster - :type master_instance_type: str - :param master_instance_type: EC2 instance type of the master - :type slave_instance_type: str - :param slave_instance_type: EC2 instance type of the slave nodes - :type num_instances: int - :param num_instances: Number of instances in the Hadoop cluster - :type action_on_failure: str - :param action_on_failure: Action to take if a step terminates - :type keep_alive: bool - :param keep_alive: Denotes whether the cluster should stay alive upon completion - :type enable_debugging: bool - :param enable_debugging: Denotes whether AWS console debugging should be enabled. - :type steps: list(boto.emr.Step) - :param steps: List of steps to add with the job - - :rtype: str - :return: The jobflow id - """ - params = {} - if action_on_failure: - params['ActionOnFailure'] = action_on_failure - params['Name'] = name - params['LogUri'] = log_uri - - # Instance args - instance_params = self._build_instance_args(ec2_keyname, availability_zone, - master_instance_type, slave_instance_type, - num_instances, keep_alive, hadoop_version) - params.update(instance_params) - - # Debugging step from EMR API docs - if enable_debugging: - debugging_step = JarStep(name='Setup Hadoop Debugging', - action_on_failure='TERMINATE_JOB_FLOW', - main_class=None, - jar=self.DebuggingJar, - step_args=self.DebuggingArgs) - steps.insert(0, debugging_step) - - # Step args - if steps: - step_args = [self._build_step_args(step) for step in steps] - params.update(self._build_step_list(step_args)) - - if bootstrap_actions: - bootstrap_action_args = [self._build_bootstrap_action_args(bootstrap_action) for bootstrap_action in bootstrap_actions] - params.update(self._build_bootstrap_action_list(bootstrap_action_args)) - - response = self.get_object('RunJobFlow', params, RunJobFlowResponse) - return response.jobflowid - - def _build_bootstrap_action_args(self, bootstrap_action): - bootstrap_action_params = {} - bootstrap_action_params['ScriptBootstrapAction.Path'] = bootstrap_action.path - - try: - bootstrap_action_params['Name'] = bootstrap_action.name - except AttributeError: - pass - - args = bootstrap_action.args() - if args: - self.build_list_params(bootstrap_action_params, args, 'ScriptBootstrapAction.Args.member') - - return bootstrap_action_params - - def _build_step_args(self, step): - step_params = {} - step_params['ActionOnFailure'] = step.action_on_failure - step_params['HadoopJarStep.Jar'] = step.jar() - - main_class = step.main_class() - if main_class: - step_params['HadoopJarStep.MainClass'] = main_class - - args = step.args() - if args: - self.build_list_params(step_params, args, 'HadoopJarStep.Args.member') - - step_params['Name'] = step.name - return step_params - - def _build_bootstrap_action_list(self, bootstrap_actions): - if type(bootstrap_actions) != types.ListType: - bootstrap_actions = [bootstrap_actions] - - params = {} - for i, bootstrap_action in enumerate(bootstrap_actions): - for key, value in bootstrap_action.iteritems(): - params['BootstrapActions.member.%s.%s' % (i + 1, key)] = value - return params - - def _build_step_list(self, steps): - if type(steps) != types.ListType: - steps = [steps] - - params = {} - for i, step in enumerate(steps): - for key, value in step.iteritems(): - params['Steps.member.%s.%s' % (i+1, key)] = value - return params - - def _build_instance_args(self, ec2_keyname, availability_zone, master_instance_type, - slave_instance_type, num_instances, keep_alive, hadoop_version): - params = { - 'Instances.MasterInstanceType' : master_instance_type, - 'Instances.SlaveInstanceType' : slave_instance_type, - 'Instances.InstanceCount' : num_instances, - 'Instances.KeepJobFlowAliveWhenNoSteps' : str(keep_alive).lower(), - 'Instances.HadoopVersion' : hadoop_version - } - - if ec2_keyname: - params['Instances.Ec2KeyName'] = ec2_keyname - if availability_zone: - params['Placement'] = availability_zone - - return params - diff --git a/backup/src/boto/emr/emrobject.py b/backup/src/boto/emr/emrobject.py deleted file mode 100644 index 0ffe292..0000000 --- a/backup/src/boto/emr/emrobject.py +++ /dev/null @@ -1,141 +0,0 @@ -# Copyright (c) 2010 Spotify AB -# Copyright (c) 2010 Jeremy Thurgood -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -This module contains EMR response objects -""" - -from boto.resultset import ResultSet - - -class EmrObject(object): - Fields = set() - - def __init__(self, connection=None): - self.connection = connection - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name in self.Fields: - setattr(self, name.lower(), value) - - -class RunJobFlowResponse(EmrObject): - Fields = set(['JobFlowId']) - - -class Arg(EmrObject): - def __init__(self, connection=None): - self.value = None - - def endElement(self, name, value, connection): - self.value = value - - -class BootstrapAction(EmrObject): - Fields = set(['Name', - 'Args', - 'Path']) - - -class Step(EmrObject): - Fields = set(['Name', - 'ActionOnFailure', - 'CreationDateTime', - 'StartDateTime', - 'EndDateTime', - 'LastStateChangeReason', - 'State']) - - def __init__(self, connection=None): - self.connection = connection - self.args = None - - def startElement(self, name, attrs, connection): - if name == 'Args': - self.args = ResultSet([('member', Arg)]) - return self.args - - -class InstanceGroup(EmrObject): - Fields = set(['Name', - 'CreationDateTime', - 'InstanceRunningCount', - 'StartDateTime', - 'ReadyDateTime', - 'State', - 'EndDateTime', - 'InstanceRequestCount', - 'InstanceType', - 'Market', - 'LastStateChangeReason', - 'InstanceRole', - 'InstanceGroupId', - 'LaunchGroup', - 'SpotPrice']) - - -class JobFlow(EmrObject): - Fields = set(['CreationDateTime', - 'StartDateTime', - 'State', - 'EndDateTime', - 'Id', - 'InstanceCount', - 'JobFlowId', - 'LogUri', - 'MasterPublicDnsName', - 'MasterInstanceId', - 'Name', - 'Placement', - 'RequestId', - 'Type', - 'Value', - 'AvailabilityZone', - 'SlaveInstanceType', - 'MasterInstanceType', - 'Ec2KeyName', - 'InstanceCount', - 'KeepJobFlowAliveWhenNoSteps', - 'LastStateChangeReason']) - - def __init__(self, connection=None): - self.connection = connection - self.steps = None - self.instancegroups = None - self.bootstrapactions = None - - def startElement(self, name, attrs, connection): - if name == 'Steps': - self.steps = ResultSet([('member', Step)]) - return self.steps - elif name == 'InstanceGroups': - self.instancegroups = ResultSet([('member', InstanceGroup)]) - return self.instancegroups - elif name == 'BootstrapActions': - self.bootstrapactions = ResultSet([('member', BootstrapAction)]) - return self.bootstrapactions - else: - return None - diff --git a/backup/src/boto/emr/step.py b/backup/src/boto/emr/step.py deleted file mode 100644 index a444261..0000000 --- a/backup/src/boto/emr/step.py +++ /dev/null @@ -1,179 +0,0 @@ -# Copyright (c) 2010 Spotify AB -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Step(object): - """ - Jobflow Step base class - """ - def jar(self): - """ - :rtype: str - :return: URI to the jar - """ - raise NotImplemented() - - def args(self): - """ - :rtype: list(str) - :return: List of arguments for the step - """ - raise NotImplemented() - - def main_class(self): - """ - :rtype: str - :return: The main class name - """ - raise NotImplemented() - - -class JarStep(Step): - """ - Custom jar step - """ - def __init__(self, name, jar, main_class=None, - action_on_failure='TERMINATE_JOB_FLOW', step_args=None): - """ - A elastic mapreduce step that executes a jar - - :type name: str - :param name: The name of the step - :type jar: str - :param jar: S3 URI to the Jar file - :type main_class: str - :param main_class: The class to execute in the jar - :type action_on_failure: str - :param action_on_failure: An action, defined in the EMR docs to take on failure. - :type step_args: list(str) - :param step_args: A list of arguments to pass to the step - """ - self.name = name - self._jar = jar - self._main_class = main_class - self.action_on_failure = action_on_failure - - if isinstance(step_args, basestring): - step_args = [step_args] - - self.step_args = step_args - - def jar(self): - return self._jar - - def args(self): - args = [] - - if self.step_args: - args.extend(self.step_args) - - return args - - def main_class(self): - return self._main_class - - -class StreamingStep(Step): - """ - Hadoop streaming step - """ - def __init__(self, name, mapper, reducer=None, - action_on_failure='TERMINATE_JOB_FLOW', - cache_files=None, cache_archives=None, - step_args=None, input=None, output=None): - """ - A hadoop streaming elastic mapreduce step - - :type name: str - :param name: The name of the step - :type mapper: str - :param mapper: The mapper URI - :type reducer: str - :param reducer: The reducer URI - :type action_on_failure: str - :param action_on_failure: An action, defined in the EMR docs to take on failure. - :type cache_files: list(str) - :param cache_files: A list of cache files to be bundled with the job - :type cache_archives: list(str) - :param cache_archives: A list of jar archives to be bundled with the job - :type step_args: list(str) - :param step_args: A list of arguments to pass to the step - :type input: str or a list of str - :param input: The input uri - :type output: str - :param output: The output uri - """ - self.name = name - self.mapper = mapper - self.reducer = reducer - self.action_on_failure = action_on_failure - self.cache_files = cache_files - self.cache_archives = cache_archives - self.input = input - self.output = output - - if isinstance(step_args, basestring): - step_args = [step_args] - - self.step_args = step_args - - def jar(self): - return '/home/hadoop/contrib/streaming/hadoop-0.18-streaming.jar' - - def main_class(self): - return None - - def args(self): - args = ['-mapper', self.mapper] - - if self.reducer: - args.extend(['-reducer', self.reducer]) - - if self.input: - if isinstance(self.input, list): - for input in self.input: - args.extend(('-input', input)) - else: - args.extend(('-input', self.input)) - if self.output: - args.extend(('-output', self.output)) - - if self.cache_files: - for cache_file in self.cache_files: - args.extend(('-cacheFile', cache_file)) - - if self.cache_archives: - for cache_archive in self.cache_archives: - args.extend(('-cacheArchive', cache_archive)) - - if self.step_args: - args.extend(self.step_args) - - if not self.reducer: - args.extend(['-jobconf', 'mapred.reduce.tasks=0']) - - return args - - def __repr__(self): - return '%s.%s(name=%r, mapper=%r, reducer=%r, action_on_failure=%r, cache_files=%r, cache_archives=%r, step_args=%r, input=%r, output=%r)' % ( - self.__class__.__module__, self.__class__.__name__, - self.name, self.mapper, self.reducer, self.action_on_failure, - self.cache_files, self.cache_archives, self.step_args, - self.input, self.output) diff --git a/backup/src/boto/emr/tests/test_emr_responses.py b/backup/src/boto/emr/tests/test_emr_responses.py deleted file mode 100644 index 77ec494..0000000 --- a/backup/src/boto/emr/tests/test_emr_responses.py +++ /dev/null @@ -1,373 +0,0 @@ -# Copyright (c) 2010 Jeremy Thurgood -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -# NOTE: These tests only cover the very simple cases I needed to test -# for the InstanceGroup fix. - -import xml.sax -import unittest - -from boto import handler -from boto.emr import emrobject -from boto.resultset import ResultSet - - -JOB_FLOW_EXAMPLE = """ - - - - - - 2009-01-28T21:49:16Z - 2009-01-28T21:49:16Z - STARTING - - MyJobFlowName - mybucket/subdir/ - - - - 2009-01-28T21:49:16Z - PENDING - - - - MyJarFile - MyMailClass - - arg1 - arg2 - - - - MyStepName - CONTINUE - - - - j-3UN6WX5RRO2AG - - - us-east-1a - - m1.small - m1.small - myec2keyname - 4 - true - - - - - - 9cea3229-ed85-11dd-9877-6fad448a8419 - - -""" - -JOB_FLOW_COMPLETED = """ - - - - - - 2010-10-21T01:00:25Z - Steps completed - 2010-10-21T01:03:59Z - 2010-10-21T01:03:59Z - COMPLETED - 2010-10-21T01:44:18Z - - - RealJobFlowName - s3n://example.emrtest.scripts/jobflow_logs/ - - - - - s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar - - s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch - - - - Setup Hadoop Debugging - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:03:59Z - COMPLETED - 2010-10-21T01:04:22Z - - - - - - /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar - - -mapper - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py - -reducer - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py - -input - s3://example.emrtest.data/raw/2010/10/20/* - -input - s3://example.emrtest.data/raw/2010/10/19/* - -input - s3://example.emrtest.data/raw/2010/10/18/* - -input - s3://example.emrtest.data/raw/2010/10/17/* - -input - s3://example.emrtest.data/raw/2010/10/16/* - -input - s3://example.emrtest.data/raw/2010/10/15/* - -input - s3://example.emrtest.data/raw/2010/10/14/* - -output - s3://example.emrtest.crunched/ - - - - testjob_Initial - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:04:22Z - COMPLETED - 2010-10-21T01:36:18Z - - - - - - /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar - - -mapper - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py - -reducer - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py - -input - s3://example.emrtest.crunched/* - -output - s3://example.emrtest.step1/ - - - - testjob_step1 - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:36:18Z - COMPLETED - 2010-10-21T01:37:51Z - - - - - - /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar - - -mapper - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py - -reducer - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py - -input - s3://example.emrtest.crunched/* - -output - s3://example.emrtest.step2/ - - - - testjob_step2 - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:37:51Z - COMPLETED - 2010-10-21T01:39:32Z - - - - - - /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar - - -mapper - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py - -reducer - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py - -input - s3://example.emrtest.step1/* - -output - s3://example.emrtest.step3/ - - - - testjob_step3 - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:39:32Z - COMPLETED - 2010-10-21T01:41:22Z - - - - - - /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar - - -mapper - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py - -reducer - s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py - -input - s3://example.emrtest.step1/* - -output - s3://example.emrtest.step4/ - - - - testjob_step4 - TERMINATE_JOB_FLOW - - - 2010-10-21T01:00:25Z - 2010-10-21T01:41:22Z - COMPLETED - 2010-10-21T01:43:03Z - - - - j-3H3Q13JPFLU22 - - m1.large - i-64c21609 - - us-east-1b - - - - 2010-10-21T01:00:25Z - 0 - 2010-10-21T01:02:09Z - 2010-10-21T01:03:03Z - ENDED - 2010-10-21T01:44:18Z - 1 - m1.large - ON_DEMAND - Job flow terminated - MASTER - ig-EVMHOZJ2SCO8 - master - - - 2010-10-21T01:00:25Z - 0 - 2010-10-21T01:03:59Z - 2010-10-21T01:03:59Z - ENDED - 2010-10-21T01:44:18Z - 9 - m1.large - ON_DEMAND - Job flow terminated - CORE - ig-YZHDYVITVHKB - slave - - - 40 - 0.20 - m1.large - ec2-184-72-153-139.compute-1.amazonaws.com - myubersecurekey - 10 - false - - - - - - c31e701d-dcb4-11df-b5d9-337fc7fe4773 - - -""" - - -class TestEMRResponses(unittest.TestCase): - def _parse_xml(self, body, markers): - rs = ResultSet(markers) - h = handler.XmlHandler(rs, None) - xml.sax.parseString(body, h) - return rs - - def _assert_fields(self, response, **fields): - for field, expected in fields.items(): - actual = getattr(response, field) - self.assertEquals(expected, actual, - "Field %s: %r != %r" % (field, expected, actual)) - - def test_JobFlows_example(self): - [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE, - [('member', emrobject.JobFlow)]) - self._assert_fields(jobflow, - creationdatetime='2009-01-28T21:49:16Z', - startdatetime='2009-01-28T21:49:16Z', - state='STARTING', - instancecount='4', - jobflowid='j-3UN6WX5RRO2AG', - loguri='mybucket/subdir/', - name='MyJobFlowName', - availabilityzone='us-east-1a', - slaveinstancetype='m1.small', - masterinstancetype='m1.small', - ec2keyname='myec2keyname', - keepjobflowalivewhennosteps='true') - - def test_JobFlows_completed(self): - [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED, - [('member', emrobject.JobFlow)]) - self._assert_fields(jobflow, - creationdatetime='2010-10-21T01:00:25Z', - startdatetime='2010-10-21T01:03:59Z', - enddatetime='2010-10-21T01:44:18Z', - state='COMPLETED', - instancecount='10', - jobflowid='j-3H3Q13JPFLU22', - loguri='s3n://example.emrtest.scripts/jobflow_logs/', - name='RealJobFlowName', - availabilityzone='us-east-1b', - slaveinstancetype='m1.large', - masterinstancetype='m1.large', - ec2keyname='myubersecurekey', - keepjobflowalivewhennosteps='false') - self.assertEquals(6, len(jobflow.steps)) - self.assertEquals(2, len(jobflow.instancegroups)) - diff --git a/backup/src/boto/exception.py b/backup/src/boto/exception.py deleted file mode 100644 index 718be46..0000000 --- a/backup/src/boto/exception.py +++ /dev/null @@ -1,430 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Exception classes - Subclassing allows you to check for specific errors -""" -import base64 -import xml.sax -from boto import handler -from boto.resultset import ResultSet - - -class BotoClientError(StandardError): - """ - General Boto Client error (error accessing AWS) - """ - - def __init__(self, reason): - StandardError.__init__(self) - self.reason = reason - - def __repr__(self): - return 'BotoClientError: %s' % self.reason - - def __str__(self): - return 'BotoClientError: %s' % self.reason - -class SDBPersistenceError(StandardError): - - pass - -class StoragePermissionsError(BotoClientError): - """ - Permissions error when accessing a bucket or key on a storage service. - """ - pass - -class S3PermissionsError(StoragePermissionsError): - """ - Permissions error when accessing a bucket or key on S3. - """ - pass - -class GSPermissionsError(StoragePermissionsError): - """ - Permissions error when accessing a bucket or key on GS. - """ - pass - -class BotoServerError(StandardError): - - def __init__(self, status, reason, body=None): - StandardError.__init__(self) - self.status = status - self.reason = reason - self.body = body or '' - self.request_id = None - self.error_code = None - self.error_message = None - self.box_usage = None - - # Attempt to parse the error response. If body isn't present, - # then just ignore the error response. - if self.body: - try: - h = handler.XmlHandler(self, self) - xml.sax.parseString(self.body, h) - except xml.sax.SAXParseException, pe: - # Go ahead and clean up anything that may have - # managed to get into the error data so we - # don't get partial garbage. - print "Warning: failed to parse error message from AWS: %s" % pe - self._cleanupParsedProperties() - - def __getattr__(self, name): - if name == 'message': - return self.error_message - if name == 'code': - return self.error_code - raise AttributeError - - def __repr__(self): - return '%s: %s %s\n%s' % (self.__class__.__name__, - self.status, self.reason, self.body) - - def __str__(self): - return '%s: %s %s\n%s' % (self.__class__.__name__, - self.status, self.reason, self.body) - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name in ('RequestId', 'RequestID'): - self.request_id = value - elif name == 'Code': - self.error_code = value - elif name == 'Message': - self.error_message = value - elif name == 'BoxUsage': - self.box_usage = value - return None - - def _cleanupParsedProperties(self): - self.request_id = None - self.error_code = None - self.error_message = None - self.box_usage = None - -class ConsoleOutput: - - def __init__(self, parent=None): - self.parent = parent - self.instance_id = None - self.timestamp = None - self.comment = None - self.output = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'instanceId': - self.instance_id = value - elif name == 'output': - self.output = base64.b64decode(value) - else: - setattr(self, name, value) - -class StorageCreateError(BotoServerError): - """ - Error creating a bucket or key on a storage service. - """ - def __init__(self, status, reason, body=None): - self.bucket = None - BotoServerError.__init__(self, status, reason, body) - - def endElement(self, name, value, connection): - if name == 'BucketName': - self.bucket = value - else: - return BotoServerError.endElement(self, name, value, connection) - -class S3CreateError(StorageCreateError): - """ - Error creating a bucket or key on S3. - """ - pass - -class GSCreateError(StorageCreateError): - """ - Error creating a bucket or key on GS. - """ - pass - -class StorageCopyError(BotoServerError): - """ - Error copying a key on a storage service. - """ - pass - -class S3CopyError(StorageCopyError): - """ - Error copying a key on S3. - """ - pass - -class GSCopyError(StorageCopyError): - """ - Error copying a key on GS. - """ - pass - -class SQSError(BotoServerError): - """ - General Error on Simple Queue Service. - """ - def __init__(self, status, reason, body=None): - self.detail = None - self.type = None - BotoServerError.__init__(self, status, reason, body) - - def startElement(self, name, attrs, connection): - return BotoServerError.startElement(self, name, attrs, connection) - - def endElement(self, name, value, connection): - if name == 'Detail': - self.detail = value - elif name == 'Type': - self.type = value - else: - return BotoServerError.endElement(self, name, value, connection) - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - for p in ('detail', 'type'): - setattr(self, p, None) - -class SQSDecodeError(BotoClientError): - """ - Error when decoding an SQS message. - """ - def __init__(self, reason, message): - BotoClientError.__init__(self, reason) - self.message = message - - def __repr__(self): - return 'SQSDecodeError: %s' % self.reason - - def __str__(self): - return 'SQSDecodeError: %s' % self.reason - -class StorageResponseError(BotoServerError): - """ - Error in response from a storage service. - """ - def __init__(self, status, reason, body=None): - self.resource = None - BotoServerError.__init__(self, status, reason, body) - - def startElement(self, name, attrs, connection): - return BotoServerError.startElement(self, name, attrs, connection) - - def endElement(self, name, value, connection): - if name == 'Resource': - self.resource = value - else: - return BotoServerError.endElement(self, name, value, connection) - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - for p in ('resource'): - setattr(self, p, None) - -class S3ResponseError(StorageResponseError): - """ - Error in response from S3. - """ - pass - -class GSResponseError(StorageResponseError): - """ - Error in response from GS. - """ - pass - -class EC2ResponseError(BotoServerError): - """ - Error in response from EC2. - """ - - def __init__(self, status, reason, body=None): - self.errors = None - self._errorResultSet = [] - BotoServerError.__init__(self, status, reason, body) - self.errors = [ (e.error_code, e.error_message) \ - for e in self._errorResultSet ] - if len(self.errors): - self.error_code, self.error_message = self.errors[0] - - def startElement(self, name, attrs, connection): - if name == 'Errors': - self._errorResultSet = ResultSet([('Error', _EC2Error)]) - return self._errorResultSet - else: - return None - - def endElement(self, name, value, connection): - if name == 'RequestID': - self.request_id = value - else: - return None # don't call subclass here - - def _cleanupParsedProperties(self): - BotoServerError._cleanupParsedProperties(self) - self._errorResultSet = [] - for p in ('errors'): - setattr(self, p, None) - -class EmrResponseError(BotoServerError): - """ - Error in response from EMR - """ - pass - -class _EC2Error: - - def __init__(self, connection=None): - self.connection = connection - self.error_code = None - self.error_message = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Code': - self.error_code = value - elif name == 'Message': - self.error_message = value - else: - return None - -class SDBResponseError(BotoServerError): - """ - Error in responses from SDB. - """ - pass - -class AWSConnectionError(BotoClientError): - """ - General error connecting to Amazon Web Services. - """ - pass - -class StorageDataError(BotoClientError): - """ - Error receiving data from a storage service. - """ - pass - -class S3DataError(StorageDataError): - """ - Error receiving data from S3. - """ - pass - -class GSDataError(StorageDataError): - """ - Error receiving data from GS. - """ - pass - -class FPSResponseError(BotoServerError): - pass - -class InvalidUriError(Exception): - """Exception raised when URI is invalid.""" - - def __init__(self, message): - Exception.__init__(self) - self.message = message - -class InvalidAclError(Exception): - """Exception raised when ACL XML is invalid.""" - - def __init__(self, message): - Exception.__init__(self) - self.message = message - -class NoAuthHandlerFound(Exception): - """Is raised when no auth handlers were found ready to authenticate.""" - pass - -class TooManyAuthHandlerReadyToAuthenticate(Exception): - """Is raised when there are more than one auth handler ready. - - In normal situation there should only be one auth handler that is ready to - authenticate. In case where more than one auth handler is ready to - authenticate, we raise this exception, to prevent unpredictable behavior - when multiple auth handlers can handle a particular case and the one chosen - depends on the order they were checked. - """ - pass - -# Enum class for resumable upload failure disposition. -class ResumableTransferDisposition(object): - # START_OVER means an attempt to resume an existing transfer failed, - # and a new resumable upload should be attempted (without delay). - START_OVER = 'START_OVER' - - # WAIT_BEFORE_RETRY means the resumable transfer failed but that it can - # be retried after a time delay. - WAIT_BEFORE_RETRY = 'WAIT_BEFORE_RETRY' - - # ABORT means the resumable transfer failed and that delaying/retrying - # within the current process will not help. - ABORT = 'ABORT' - -class ResumableUploadException(Exception): - """ - Exception raised for various resumable upload problems. - - self.disposition is of type ResumableTransferDisposition. - """ - - def __init__(self, message, disposition): - Exception.__init__(self) - self.message = message - self.disposition = disposition - - def __repr__(self): - return 'ResumableUploadException("%s", %s)' % ( - self.message, self.disposition) - -class ResumableDownloadException(Exception): - """ - Exception raised for various resumable download problems. - - self.disposition is of type ResumableTransferDisposition. - """ - - def __init__(self, message, disposition): - Exception.__init__(self) - self.message = message - self.disposition = disposition - - def __repr__(self): - return 'ResumableDownloadException("%s", %s)' % ( - self.message, self.disposition) diff --git a/backup/src/boto/file/README b/backup/src/boto/file/README deleted file mode 100644 index af82455..0000000 --- a/backup/src/boto/file/README +++ /dev/null @@ -1,49 +0,0 @@ -Handling of file:// URIs: - -This directory contains code to map basic boto connection, bucket, and key -operations onto files in the local filesystem, in support of file:// -URI operations. - -Bucket storage operations cannot be mapped completely onto a file system -because of the different naming semantics in these types of systems: the -former have a flat name space of objects within each named bucket; the -latter have a hierarchical name space of files, and nothing corresponding to -the notion of a bucket. The mapping we selected was guided by the desire -to achieve meaningful semantics for a useful subset of operations that can -be implemented polymorphically across both types of systems. We considered -several possibilities for mapping path names to bucket + object name: - -1) bucket = the file system root or local directory (for absolute vs -relative file:// URIs, respectively) and object = remainder of path. -We discarded this choice because the get_all_keys() method doesn't make -sense under this approach: Enumerating all files under the root or current -directory could include more than the caller intended. For example, -StorageUri("file:///usr/bin/X11/vim").get_all_keys() would enumerate all -files in the file system. - -2) bucket is treated mostly as an anonymous placeholder, with the object -name holding the URI path (minus the "file://" part). Two sub-options, -for object enumeration (the get_all_keys() call): - a) disallow get_all_keys(). This isn't great, as then the caller must - know the URI type before deciding whether to make this call. - b) return the single key for which this "bucket" was defined. - Note that this option means the app cannot use this API for listing - contents of the file system. While that makes the API less generally - useful, it avoids the potentially dangerous/unintended consequences - noted in option (1) above. - -We selected 2b, resulting in a class hierarchy where StorageUri is an abstract -class, with FileStorageUri and BucketStorageUri subclasses. - -Some additional notes: - -BucketStorageUri and FileStorageUri each implement these methods: - - clone_replace_name() creates a same-type URI with a - different object name - which is useful for various enumeration cases - (e.g., implementing wildcarding in a command line utility). - - names_container() determines if the given URI names a container for - multiple objects/files - i.e., a bucket or directory. - - names_singleton() determines if the given URI names an individual object - or file. - - is_file_uri() and is_cloud_uri() determine if the given URI is a - FileStorageUri or BucketStorageUri, respectively diff --git a/backup/src/boto/file/__init__.py b/backup/src/boto/file/__init__.py deleted file mode 100644 index 0210b47..0000000 --- a/backup/src/boto/file/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto - -from connection import FileConnection as Connection -from key import Key -from bucket import Bucket - -__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/backup/src/boto/file/bucket.py b/backup/src/boto/file/bucket.py deleted file mode 100644 index 7a1636b..0000000 --- a/backup/src/boto/file/bucket.py +++ /dev/null @@ -1,101 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# File representation of bucket, for use with "file://" URIs. - -import os -from key import Key -from boto.file.simpleresultset import SimpleResultSet -from boto.s3.bucketlistresultset import BucketListResultSet - -class Bucket(object): - def __init__(self, name, contained_key): - """Instantiate an anonymous file-based Bucket around a single key. - """ - self.name = name - self.contained_key = contained_key - - def __iter__(self): - return iter(BucketListResultSet(self)) - - def __str__(self): - return 'anonymous bucket for file://' + self.contained_key - - def delete_key(self, key_name, headers=None, - version_id=None, mfa_token=None): - """ - Deletes a key from the bucket. - - :type key_name: string - :param key_name: The key name to delete - - :type version_id: string - :param version_id: Unused in this subclass. - - :type mfa_token: tuple or list of strings - :param mfa_token: Unused in this subclass. - """ - os.remove(key_name) - - def get_all_keys(self, headers=None, **params): - """ - This method returns the single key around which this anonymous Bucket - was instantiated. - - :rtype: SimpleResultSet - :return: The result from file system listing the keys requested - - """ - key = Key(self.name, self.contained_key) - return SimpleResultSet([key]) - - def get_key(self, key_name, headers=None, version_id=None): - """ - Check to see if a particular key exists within the bucket. - Returns: An instance of a Key object or None - - :type key_name: string - :param key_name: The name of the key to retrieve - - :type version_id: string - :param version_id: Unused in this subclass. - - :rtype: :class:`boto.file.key.Key` - :returns: A Key object from this bucket. - """ - fp = open(key_name, 'rb') - return Key(self.name, key_name, fp) - - def new_key(self, key_name=None): - """ - Creates a new key - - :type key_name: string - :param key_name: The name of the key to create - - :rtype: :class:`boto.file.key.Key` - :returns: An instance of the newly created key object - """ - dir_name = os.path.dirname(key_name) - if dir_name and not os.path.exists(dir_name): - os.makedirs(dir_name) - fp = open(key_name, 'wb') - return Key(self.name, key_name, fp) diff --git a/backup/src/boto/file/connection.py b/backup/src/boto/file/connection.py deleted file mode 100644 index f453f71..0000000 --- a/backup/src/boto/file/connection.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# File representation of connection, for use with "file://" URIs. - -from bucket import Bucket - -class FileConnection(object): - - def __init__(self, file_storage_uri): - # FileConnections are per-file storage URI. - self.file_storage_uri = file_storage_uri - - def get_bucket(self, bucket_name, validate=True, headers=None): - return Bucket(bucket_name, self.file_storage_uri.object_name) diff --git a/backup/src/boto/file/key.py b/backup/src/boto/file/key.py deleted file mode 100644 index af801a5..0000000 --- a/backup/src/boto/file/key.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# File representation of key, for use with "file://" URIs. - -import os, shutil, StringIO - -class Key(object): - - def __init__(self, bucket, name, fp=None): - self.bucket = bucket - self.full_path = name - self.name = name - self.fp = fp - - def __str__(self): - return 'file://' + self.full_path - - def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): - """ - Retrieves a file from a Key - - :type fp: file - :param fp: File pointer to put the data into - - :type headers: string - :param: ignored in this subclass. - - :type cb: function - :param cb: ignored in this subclass. - - :type cb: int - :param num_cb: ignored in this subclass. - """ - key_file = open(self.full_path, 'rb') - shutil.copyfileobj(key_file, fp) - - def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, - num_cb=10, policy=None, md5=None): - """ - Store an object in a file using the name of the Key object as the - key in file URI and the contents of the file pointed to by 'fp' as the - contents. - - :type fp: file - :param fp: the file whose contents to upload - - :type headers: dict - :param headers: ignored in this subclass. - - :type replace: bool - :param replace: If this parameter is False, the method - will first check to see if an object exists in the - bucket with the same key. If it does, it won't - overwrite it. The default value is True which will - overwrite the object. - - :type cb: function - :param cb: ignored in this subclass. - - :type cb: int - :param num_cb: ignored in this subclass. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: ignored in this subclass. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum - of the file as the first element and the Base64-encoded - version of the plain checksum as the second element. - This is the same format returned by the compute_md5 method. - :param md5: ignored in this subclass. - """ - if not replace and os.path.exists(self.full_path): - return - key_file = open(self.full_path, 'wb') - shutil.copyfileobj(fp, key_file) - key_file.close() - - def get_contents_as_string(self, headers=None, cb=None, num_cb=10, - torrent=False): - """ - Retrieve file data from the Key, and return contents as a string. - - :type headers: dict - :param headers: ignored in this subclass. - - :type cb: function - :param cb: ignored in this subclass. - - :type cb: int - :param num_cb: ignored in this subclass. - - :type cb: int - :param num_cb: ignored in this subclass. - - :type torrent: bool - :param torrent: ignored in this subclass. - - :rtype: string - :returns: The contents of the file as a string - """ - - fp = StringIO.StringIO() - self.get_contents_to_file(fp) - return fp.getvalue() diff --git a/backup/src/boto/file/simpleresultset.py b/backup/src/boto/file/simpleresultset.py deleted file mode 100644 index 5f94dc1..0000000 --- a/backup/src/boto/file/simpleresultset.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class SimpleResultSet(list): - """ - ResultSet facade built from a simple list, rather than via XML parsing. - """ - - def __init__(self, input_list): - for x in input_list: - self.append(x) - self.is_truncated = False diff --git a/backup/src/boto/fps/__init__.py b/backup/src/boto/fps/__init__.py deleted file mode 100644 index 2f44483..0000000 --- a/backup/src/boto/fps/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2008, Chris Moyer http://coredumped.org -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - - diff --git a/backup/src/boto/fps/connection.py b/backup/src/boto/fps/connection.py deleted file mode 100644 index 3d7812e..0000000 --- a/backup/src/boto/fps/connection.py +++ /dev/null @@ -1,356 +0,0 @@ -# Copyright (c) 2008 Chris Moyer http://coredumped.org/ -# Copyringt (c) 2010 Jason R. Coombs http://www.jaraco.com/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import base64 -import hmac -import hashlib -import urllib -import xml.sax -import uuid -import boto -import boto.utils -from boto import handler -from boto.connection import AWSQueryConnection -from boto.resultset import ResultSet -from boto.exception import FPSResponseError - -class FPSConnection(AWSQueryConnection): - - APIVersion = '2007-01-08' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, - host='fps.sandbox.amazonaws.com', debug=0, - https_connection_factory=None, path="/"): - AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, host, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['fps'] - - def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): - """ - InstallPaymentInstruction - instruction: The PaymentInstruction to send, for example: - - MyRole=='Caller' orSay 'Roles do not match'; - - token_type: Defaults to "Unrestricted" - transaction_id: Defaults to a new ID - """ - - if(transaction_id == None): - transaction_id = uuid.uuid4() - params = {} - params['PaymentInstruction'] = instruction - params['TokenType'] = token_type - params['CallerReference'] = transaction_id - response = self.make_request("InstallPaymentInstruction", params) - return response - - def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): - """ - Set us up as a caller - This will install a new caller_token into the FPS section. - This should really only be called to regenerate the caller token. - """ - response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - caller_token = rs.TokenId - try: - boto.config.save_system_option("FPS", "caller_token", caller_token) - except(IOError): - boto.config.save_user_option("FPS", "caller_token", caller_token) - return caller_token - else: - raise FPSResponseError(response.status, response.reason, body) - - def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): - """ - Set us up as a Recipient - This will install a new caller_token into the FPS section. - This should really only be called to regenerate the recipient token. - """ - response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - recipient_token = rs.TokenId - try: - boto.config.save_system_option("FPS", "recipient_token", recipient_token) - except(IOError): - boto.config.save_user_option("FPS", "recipient_token", recipient_token) - - return recipient_token - else: - raise FPSResponseError(response.status, response.reason, body) - - def make_url(self, returnURL, paymentReason, pipelineName, transactionAmount, **params): - """ - Generate the URL with the signature required for a transaction - """ - # use the sandbox authorization endpoint if we're using the - # sandbox for API calls. - endpoint_host = 'authorize.payments.amazon.com' - if 'sandbox' in self.host: - endpoint_host = 'authorize.payments-sandbox.amazon.com' - base = "/cobranded-ui/actions/start" - - - params['callerKey'] = str(self.aws_access_key_id) - params['returnURL'] = str(returnURL) - params['paymentReason'] = str(paymentReason) - params['pipelineName'] = pipelineName - params["signatureMethod"] = 'HmacSHA256' - params["signatureVersion"] = '2' - params["transactionAmount"] = transactionAmount - - if(not params.has_key('callerReference')): - params['callerReference'] = str(uuid.uuid4()) - - parts = '' - for k in sorted(params.keys()): - parts += "&%s=%s" % (k, urllib.quote(params[k], '~')) - - canonical = '\n'.join(['GET', - str(endpoint_host).lower(), - base, - parts[1:]]) - - signature = self._auth_handler.sign_string(canonical) - params["signature"] = signature - - urlsuffix = '' - for k in sorted(params.keys()): - urlsuffix += "&%s=%s" % (k, urllib.quote(params[k], '~')) - urlsuffix = urlsuffix[1:] # strip the first & - - fmt = "https://%(endpoint_host)s%(base)s?%(urlsuffix)s" - final = fmt % vars() - return final - - def pay(self, transactionAmount, senderTokenId, - recipientTokenId=None, callerTokenId=None, - chargeFeeTo="Recipient", - callerReference=None, senderReference=None, recipientReference=None, - senderDescription=None, recipientDescription=None, callerDescription=None, - metadata=None, transactionDate=None, reserve=False): - """ - Make a payment transaction. You must specify the amount. - This can also perform a Reserve request if 'reserve' is set to True. - """ - params = {} - params['SenderTokenId'] = senderTokenId - # this is for 2008-09-17 specification - params['TransactionAmount.Amount'] = str(transactionAmount) - params['TransactionAmount.CurrencyCode'] = "USD" - #params['TransactionAmount'] = str(transactionAmount) - params['ChargeFeeTo'] = chargeFeeTo - - params['RecipientTokenId'] = ( - recipientTokenId if recipientTokenId is not None - else boto.config.get("FPS", "recipient_token") - ) - params['CallerTokenId'] = ( - callerTokenId if callerTokenId is not None - else boto.config.get("FPS", "caller_token") - ) - if(transactionDate != None): - params['TransactionDate'] = transactionDate - if(senderReference != None): - params['SenderReference'] = senderReference - if(recipientReference != None): - params['RecipientReference'] = recipientReference - if(senderDescription != None): - params['SenderDescription'] = senderDescription - if(recipientDescription != None): - params['RecipientDescription'] = recipientDescription - if(callerDescription != None): - params['CallerDescription'] = callerDescription - if(metadata != None): - params['MetaData'] = metadata - if(callerReference == None): - callerReference = uuid.uuid4() - params['CallerReference'] = callerReference - - if reserve: - response = self.make_request("Reserve", params) - else: - response = self.make_request("Pay", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def get_transaction_status(self, transactionId): - """ - Returns the status of a given transaction. - """ - params = {} - params['TransactionId'] = transactionId - - response = self.make_request("GetTransactionStatus", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def cancel(self, transactionId, description=None): - """ - Cancels a reserved or pending transaction. - """ - params = {} - params['transactionId'] = transactionId - if(description != None): - params['description'] = description - - response = self.make_request("Cancel", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def settle(self, reserveTransactionId, transactionAmount=None): - """ - Charges for a reserved payment. - """ - params = {} - params['ReserveTransactionId'] = reserveTransactionId - if(transactionAmount != None): - params['TransactionAmount'] = transactionAmount - - response = self.make_request("Settle", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def refund(self, callerReference, transactionId, refundAmount=None, callerDescription=None): - """ - Refund a transaction. This refunds the full amount by default unless 'refundAmount' is specified. - """ - params = {} - params['CallerReference'] = callerReference - params['TransactionId'] = transactionId - if(refundAmount != None): - params['RefundAmount'] = refundAmount - if(callerDescription != None): - params['CallerDescription'] = callerDescription - - response = self.make_request("Refund", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def get_recipient_verification_status(self, recipientTokenId): - """ - Test that the intended recipient has a verified Amazon Payments account. - """ - params ={} - params['RecipientTokenId'] = recipientTokenId - - response = self.make_request("GetRecipientVerificationStatus", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def get_token_by_caller_reference(self, callerReference): - """ - Returns details about the token specified by 'callerReference'. - """ - params ={} - params['callerReference'] = callerReference - - response = self.make_request("GetTokenByCaller", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - def get_token_by_caller_token(self, tokenId): - """ - Returns details about the token specified by 'callerReference'. - """ - params ={} - params['TokenId'] = tokenId - - response = self.make_request("GetTokenByCaller", params) - body = response.read() - if(response.status == 200): - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise FPSResponseError(response.status, response.reason, body) - - def verify_signature(self, end_point_url, http_parameters): - params = dict( - UrlEndPoint = end_point_url, - HttpParameters = http_parameters, - ) - response = self.make_request("VerifySignature", params) - body = response.read() - if(response.status != 200): - raise FPSResponseError(response.status, response.reason, body) - rs = ResultSet() - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs diff --git a/backup/src/boto/fps/test/__init__.py b/backup/src/boto/fps/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backup/src/boto/fps/test/test_install_caller_instruction.py b/backup/src/boto/fps/test/test_install_caller_instruction.py deleted file mode 100644 index 45df867..0000000 --- a/backup/src/boto/fps/test/test_install_caller_instruction.py +++ /dev/null @@ -1,4 +0,0 @@ -from boto.fps.connection import FPSConnection -conn = FPSConnection() -conn.install_caller_instruction() -conn.install_recipient_instruction() diff --git a/backup/src/boto/fps/test/test_verify_signature.py b/backup/src/boto/fps/test/test_verify_signature.py deleted file mode 100644 index d59b4fb..0000000 --- a/backup/src/boto/fps/test/test_verify_signature.py +++ /dev/null @@ -1,6 +0,0 @@ -from boto.fps.connection import FPSConnection -conn = FPSConnection() -# example response from the docs -params = 'expiry=08%2F2015&signature=ynDukZ9%2FG77uSJVb5YM0cadwHVwYKPMKOO3PNvgADbv6VtymgBxeOWEhED6KGHsGSvSJnMWDN%2FZl639AkRe9Ry%2F7zmn9CmiM%2FZkp1XtshERGTqi2YL10GwQpaH17MQqOX3u1cW4LlyFoLy4celUFBPq1WM2ZJnaNZRJIEY%2FvpeVnCVK8VIPdY3HMxPAkNi5zeF2BbqH%2BL2vAWef6vfHkNcJPlOuOl6jP4E%2B58F24ni%2B9ek%2FQH18O4kw%2FUJ7ZfKwjCCI13%2BcFybpofcKqddq8CuUJj5Ii7Pdw1fje7ktzHeeNhF0r9siWcYmd4JaxTP3NmLJdHFRq2T%2FgsF3vK9m3gw%3D%3D&signatureVersion=2&signatureMethod=RSA-SHA1&certificateUrl=https%3A%2F%2Ffps.sandbox.amazonaws.com%2Fcerts%2F090909%2FPKICert.pem&tokenID=A5BB3HUNAZFJ5CRXIPH72LIODZUNAUZIVP7UB74QNFQDSQ9MN4HPIKISQZWPLJXF&status=SC&callerReference=callerReferenceMultiUse1' -endpoint = 'http://vamsik.desktop.amazon.com:8080/ipn.jsp' -conn.verify_signature(endpoint, params) diff --git a/backup/src/boto/gs/__init__.py b/backup/src/boto/gs/__init__.py deleted file mode 100644 index bf4c0b9..0000000 --- a/backup/src/boto/gs/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/backup/src/boto/gs/acl.py b/backup/src/boto/gs/acl.py deleted file mode 100644 index 33aaadf..0000000 --- a/backup/src/boto/gs/acl.py +++ /dev/null @@ -1,276 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.gs.user import User -from boto.exception import InvalidAclError - -ACCESS_CONTROL_LIST = 'AccessControlList' -ALL_AUTHENTICATED_USERS = 'AllAuthenticatedUsers' -ALL_USERS = 'AllUsers' -DOMAIN = 'Domain' -EMAIL_ADDRESS = 'EmailAddress' -ENTRY = 'Entry' -ENTRIES = 'Entries' -GROUP_BY_DOMAIN = 'GroupByDomain' -GROUP_BY_EMAIL = 'GroupByEmail' -GROUP_BY_ID = 'GroupById' -ID = 'ID' -NAME = 'Name' -OWNER = 'Owner' -PERMISSION = 'Permission' -SCOPE = 'Scope' -TYPE = 'type' -USER_BY_EMAIL = 'UserByEmail' -USER_BY_ID = 'UserById' - - -CannedACLStrings = ['private', 'public-read', - 'public-read-write', 'authenticated-read', - 'bucket-owner-read', 'bucket-owner-full-control'] - -SupportedPermissions = ['READ', 'WRITE', 'FULL_CONTROL'] - -class ACL: - - def __init__(self, parent=None): - self.parent = parent - self.entries = [] - - def __repr__(self): - # Owner is optional in GS ACLs. - if hasattr(self, 'owner'): - entries_repr = [''] - else: - entries_repr = ['Owner:%s' % self.owner.__repr__()] - acl_entries = self.entries - if acl_entries: - for e in acl_entries.entry_list: - entries_repr.append(e.__repr__()) - return '<%s>' % ', '.join(entries_repr) - - # Method with same signature as boto.s3.acl.ACL.add_email_grant(), to allow - # polymorphic treatment at application layer. - def add_email_grant(self, permission, email_address): - entry = Entry(type=USER_BY_EMAIL, email_address=email_address, - permission=permission) - self.entries.entry_list.append(entry) - - # Method with same signature as boto.s3.acl.ACL.add_user_grant(), to allow - # polymorphic treatment at application layer. - def add_user_grant(self, permission, user_id): - entry = Entry(permission=permission, type=USER_BY_ID, id=user_id) - self.entries.entry_list.append(entry) - - def add_group_email_grant(self, permission, email_address): - entry = Entry(type=GROUP_BY_EMAIL, email_address=email_address, - permission=permission) - self.entries.entry_list.append(entry) - - def add_group_grant(self, permission, group_id): - entry = Entry(type=GROUP_BY_ID, id=group_id, permission=permission) - self.entries.entry_list.append(entry) - - def startElement(self, name, attrs, connection): - if name == OWNER: - self.owner = User(self) - return self.owner - elif name == ENTRIES: - self.entries = Entries(self) - return self.entries - else: - return None - - def endElement(self, name, value, connection): - if name == OWNER: - pass - elif name == ENTRIES: - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '<%s>' % ACCESS_CONTROL_LIST - # Owner is optional in GS ACLs. - if hasattr(self, 'owner'): - s += self.owner.to_xml() - acl_entries = self.entries - if acl_entries: - s += acl_entries.to_xml() - s += '' % ACCESS_CONTROL_LIST - return s - - -class Entries: - - def __init__(self, parent=None): - self.parent = parent - # Entries is the class that represents the same-named XML - # element. entry_list is the list within this class that holds the data. - self.entry_list = [] - - def __repr__(self): - entries_repr = [] - for e in self.entry_list: - entries_repr.append(e.__repr__()) - return '' % ', '.join(entries_repr) - - def startElement(self, name, attrs, connection): - if name == ENTRY: - entry = Entry(self) - self.entry_list.append(entry) - return entry - else: - return None - - def endElement(self, name, value, connection): - if name == ENTRY: - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '<%s>' % ENTRIES - for entry in self.entry_list: - s += entry.to_xml() - s += '' % ENTRIES - return s - - -# Class that represents a single (Scope, Permission) entry in an ACL. -class Entry: - - def __init__(self, scope=None, type=None, id=None, name=None, - email_address=None, domain=None, permission=None): - if not scope: - scope = Scope(self, type, id, name, email_address, domain) - self.scope = scope - self.permission = permission - - def __repr__(self): - return '<%s: %s>' % (self.scope.__repr__(), self.permission.__repr__()) - - def startElement(self, name, attrs, connection): - if name == SCOPE: - if not TYPE in attrs: - raise InvalidAclError('Missing "%s" in "%s" part of ACL' % - (TYPE, SCOPE)) - self.scope = Scope(self, attrs[TYPE]) - return self.scope - elif name == PERMISSION: - pass - else: - return None - - def endElement(self, name, value, connection): - if name == SCOPE: - pass - elif name == PERMISSION: - value = value.strip() - if not value in SupportedPermissions: - raise InvalidAclError('Invalid Permission "%s"' % value) - self.permission = value - else: - setattr(self, name, value) - - def to_xml(self): - s = '<%s>' % ENTRY - s += self.scope.to_xml() - s += '<%s>%s' % (PERMISSION, self.permission, PERMISSION) - s += '' % ENTRY - return s - -class Scope: - - # Map from Scope type to list of allowed sub-elems. - ALLOWED_SCOPE_TYPE_SUB_ELEMS = { - ALL_AUTHENTICATED_USERS : [], - ALL_USERS : [], - GROUP_BY_DOMAIN : [DOMAIN], - GROUP_BY_EMAIL : [EMAIL_ADDRESS, NAME], - GROUP_BY_ID : [ID, NAME], - USER_BY_EMAIL : [EMAIL_ADDRESS, NAME], - USER_BY_ID : [ID, NAME] - } - - def __init__(self, parent, type=None, id=None, name=None, - email_address=None, domain=None): - self.parent = parent - self.type = type - self.name = name - self.id = id - self.domain = domain - self.email_address = email_address - if not self.ALLOWED_SCOPE_TYPE_SUB_ELEMS.has_key(self.type): - raise InvalidAclError('Invalid %s %s "%s" ' % - (SCOPE, TYPE, self.type)) - - def __repr__(self): - named_entity = None - if self.id: - named_entity = self.id - elif self.email_address: - named_entity = self.email_address - elif self.domain: - named_entity = self.domain - if named_entity: - return '<%s: %s>' % (self.type, named_entity) - else: - return '<%s>' % self.type - - def startElement(self, name, attrs, connection): - if not name in self.ALLOWED_SCOPE_TYPE_SUB_ELEMS[self.type]: - raise InvalidAclError('Element "%s" not allowed in %s %s "%s" ' % - (name, SCOPE, TYPE, self.type)) - return None - - def endElement(self, name, value, connection): - value = value.strip() - if name == DOMAIN: - self.domain = value - elif name == EMAIL_ADDRESS: - self.email_address = value - elif name == ID: - self.id = value - elif name == NAME: - self.name = value - else: - setattr(self, name, value) - - def to_xml(self): - s = '<%s type="%s">' % (SCOPE, self.type) - if self.type == ALL_AUTHENTICATED_USERS or self.type == ALL_USERS: - pass - elif self.type == GROUP_BY_DOMAIN: - s += '<%s>%s' % (DOMAIN, self.domain, DOMAIN) - elif self.type == GROUP_BY_EMAIL or self.type == USER_BY_EMAIL: - s += '<%s>%s' % (EMAIL_ADDRESS, self.email_address, - EMAIL_ADDRESS) - if self.name: - s += '<%s>%s' % (NAME, self.name, NAME) - elif self.type == GROUP_BY_ID or self.type == USER_BY_ID: - s += '<%s>%s' % (ID, self.id, ID) - if self.name: - s += '<%s>%s' % (NAME, self.name, NAME) - else: - raise InvalidAclError('Invalid scope type "%s" ', self.type) - - s += '' % SCOPE - return s diff --git a/backup/src/boto/gs/bucket.py b/backup/src/boto/gs/bucket.py deleted file mode 100644 index b4b80e8..0000000 --- a/backup/src/boto/gs/bucket.py +++ /dev/null @@ -1,173 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto import handler -from boto.exception import InvalidAclError -from boto.gs.acl import ACL -from boto.gs.acl import SupportedPermissions as GSPermissions -from boto.gs.key import Key as GSKey -from boto.s3.acl import Policy -from boto.s3.bucket import Bucket as S3Bucket -import xml.sax - -class Bucket(S3Bucket): - - def __init__(self, connection=None, name=None, key_class=GSKey): - super(Bucket, self).__init__(connection, name, key_class) - - def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): - if isinstance(acl_or_str, Policy): - raise InvalidAclError('Attempt to set S3 Policy on GS ACL') - elif isinstance(acl_or_str, ACL): - self.set_xml_acl(acl_or_str.to_xml(), key_name, headers=headers) - else: - self.set_canned_acl(acl_or_str, key_name, headers=headers) - - def get_acl(self, key_name='', headers=None, version_id=None): - response = self.connection.make_request('GET', self.name, key_name, - query_args='acl', headers=headers) - body = response.read() - if response.status == 200: - acl = ACL(self) - h = handler.XmlHandler(acl, self) - xml.sax.parseString(body, h) - return acl - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - # Method with same signature as boto.s3.bucket.Bucket.add_email_grant(), - # to allow polymorphic treatment at application layer. - def add_email_grant(self, permission, email_address, - recursive=False, headers=None): - """ - Convenience method that provides a quick way to add an email grant - to a bucket. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL - and then PUT's the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, FULL_CONTROL). - - :type email_address: string - :param email_address: The email address associated with the GS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the call - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in GSPermissions: - raise self.connection.provider.storage_permissions_error( - 'Unknown Permission: %s' % permission) - acl = self.get_acl(headers=headers) - acl.add_email_grant(permission, email_address) - self.set_acl(acl, headers=headers) - if recursive: - for key in self: - key.add_email_grant(permission, email_address, headers=headers) - - # Method with same signature as boto.s3.bucket.Bucket.add_user_grant(), - # to allow polymorphic treatment at application layer. - def add_user_grant(self, permission, user_id, recursive=False, headers=None): - """ - Convenience method that provides a quick way to add a canonical user grant to a bucket. - This method retrieves the current ACL, creates a new grant based on the parameters - passed in, adds that grant to the ACL and then PUTs the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ|WRITE|FULL_CONTROL) - - :type user_id: string - :param user_id: The canonical user id associated with the GS account you are granting - the permission to. - - :type recursive: bool - :param recursive: A boolean value to controls whether the call - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in GSPermissions: - raise self.connection.provider.storage_permissions_error( - 'Unknown Permission: %s' % permission) - acl = self.get_acl(headers=headers) - acl.add_user_grant(permission, user_id) - self.set_acl(acl, headers=headers) - if recursive: - for key in self: - key.add_user_grant(permission, user_id, headers=headers) - - def add_group_email_grant(self, permission, email_address, recursive=False, - headers=None): - """ - Convenience method that provides a quick way to add an email group - grant to a bucket. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL and - then PUT's the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|WRITE|FULL_CONTROL - See http://code.google.com/apis/storage/docs/developer-guide.html#authorization - for more details on permissions. - - :type email_address: string - :param email_address: The email address associated with the Google - Group to which you are granting the permission. - - :type recursive: bool - :param recursive: A boolean value to controls whether the call - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in GSPermissions: - raise self.connection.provider.storage_permissions_error( - 'Unknown Permission: %s' % permission) - acl = self.get_acl(headers=headers) - acl.add_group_email_grant(permission, email_address) - self.set_acl(acl, headers=headers) - if recursive: - for key in self: - key.add_group_email_grant(permission, email_address, - headers=headers) - - # Method with same input signature as boto.s3.bucket.Bucket.list_grants() - # (but returning different object type), to allow polymorphic treatment - # at application layer. - def list_grants(self, headers=None): - acl = self.get_acl(headers=headers) - return acl.entries diff --git a/backup/src/boto/gs/connection.py b/backup/src/boto/gs/connection.py deleted file mode 100644 index ec81f32..0000000 --- a/backup/src/boto/gs/connection.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.s3.connection import S3Connection -from boto.s3.connection import SubdomainCallingFormat -from boto.gs.bucket import Bucket - -class GSConnection(S3Connection): - - DefaultHost = 'commondatastorage.googleapis.com' - QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' - - def __init__(self, gs_access_key_id=None, gs_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, - host=DefaultHost, debug=0, https_connection_factory=None, - calling_format=SubdomainCallingFormat(), path='/'): - S3Connection.__init__(self, gs_access_key_id, gs_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - host, debug, https_connection_factory, calling_format, path, - "google", Bucket) diff --git a/backup/src/boto/gs/key.py b/backup/src/boto/gs/key.py deleted file mode 100644 index 608a9a5..0000000 --- a/backup/src/boto/gs/key.py +++ /dev/null @@ -1,247 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.s3.key import Key as S3Key - -class Key(S3Key): - - def add_email_grant(self, permission, email_address): - """ - Convenience method that provides a quick way to add an email grant to a - key. This method retrieves the current ACL, creates a new grant based on - the parameters passed in, adds that grant to the ACL and then PUT's the - new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|FULL_CONTROL - See http://code.google.com/apis/storage/docs/developer-guide.html#authorization - for more details on permissions. - - :type email_address: string - :param email_address: The email address associated with the Google - account to which you are granting the permission. - """ - acl = self.get_acl() - acl.add_email_grant(permission, email_address) - self.set_acl(acl) - - def add_user_grant(self, permission, user_id): - """ - Convenience method that provides a quick way to add a canonical user - grant to a key. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL and - then PUT's the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|FULL_CONTROL - See http://code.google.com/apis/storage/docs/developer-guide.html#authorization - for more details on permissions. - - :type user_id: string - :param user_id: The canonical user id associated with the GS account to - which you are granting the permission. - """ - acl = self.get_acl() - acl.add_user_grant(permission, user_id) - self.set_acl(acl) - - def add_group_email_grant(self, permission, email_address, headers=None): - """ - Convenience method that provides a quick way to add an email group - grant to a key. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL and - then PUT's the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|FULL_CONTROL - See http://code.google.com/apis/storage/docs/developer-guide.html#authorization - for more details on permissions. - - :type email_address: string - :param email_address: The email address associated with the Google - Group to which you are granting the permission. - """ - acl = self.get_acl(headers=headers) - acl.add_group_email_grant(permission, email_address) - self.set_acl(acl, headers=headers) - - def add_group_grant(self, permission, group_id): - """ - Convenience method that provides a quick way to add a canonical group - grant to a key. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL and - then PUT's the new ACL back to GS. - - :type permission: string - :param permission: The permission being granted. Should be one of: - READ|FULL_CONTROL - See http://code.google.com/apis/storage/docs/developer-guide.html#authorization - for more details on permissions. - - :type group_id: string - :param group_id: The canonical group id associated with the Google - Groups account you are granting the permission to. - """ - acl = self.get_acl() - acl.add_group_grant(permission, group_id) - self.set_acl(acl) - - def set_contents_from_file(self, fp, headers={}, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - res_upload_handler=None): - """ - Store an object in GS using the name of the Key object as the - key in GS and the contents of the file pointed to by 'fp' as the - contents. - - :type fp: file - :param fp: the file whose contents are to be uploaded - - :type headers: dict - :param headers: additional HTTP headers to be sent with the PUT request. - - :type replace: bool - :param replace: If this parameter is False, the method will first check - to see if an object exists in the bucket with the same key. If it - does, it won't overwrite it. The default value is True which will - overwrite the object. - - :type cb: function - :param cb: a callback function that will be called to report - progress on the upload. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted to GS and the second representing the - total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter, this parameter determines the granularity of the callback - by defining the maximum number of times the callback will be called - during the file transfer. - - :type policy: :class:`boto.gs.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key - in GS. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum - of the file as the first element and the Base64-encoded version of - the plain checksum as the second element. This is the same format - returned by the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to - upload, it's silly to have to do it twice so this param, if present, - will be used as the MD5 values of the file. Otherwise, the checksum - will be computed. - - :type res_upload_handler: ResumableUploadHandler - :param res_upload_handler: If provided, this handler will perform the - upload. - - TODO: At some point we should refactor the Bucket and Key classes, - to move functionality common to all providers into a parent class, - and provider-specific functionality into subclasses (rather than - just overriding/sharing code the way it currently works). - """ - provider = self.bucket.connection.provider - if headers is None: - headers = {} - if policy: - headers[provider.acl_header] = policy - if hasattr(fp, 'name'): - self.path = fp.name - if self.bucket != None: - if not md5: - md5 = self.compute_md5(fp) - else: - # Even if md5 is provided, still need to set size of content. - fp.seek(0, 2) - self.size = fp.tell() - fp.seek(0) - self.md5 = md5[0] - self.base64md5 = md5[1] - if self.name == None: - self.name = self.md5 - if not replace: - k = self.bucket.lookup(self.name) - if k: - return - if res_upload_handler: - res_upload_handler.send_file(self, fp, headers, cb, num_cb) - else: - # Not a resumable transfer so use basic send_file mechanism. - self.send_file(fp, headers, cb, num_cb) - - def set_contents_from_filename(self, filename, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=None, - res_upload_handler=None): - """ - Store an object in GS using the name of the Key object as the - key in GS and the contents of the file named by 'filename'. - See set_contents_from_file method for details about the - parameters. - - :type filename: string - :param filename: The name of the file that you want to put onto GS - - :type headers: dict - :param headers: Additional headers to pass along with the request to GS. - - :type replace: bool - :param replace: If True, replaces the contents of the file if it - already exists. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from GS and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter this parameter determines the granularity of the callback - by defining the maximum number of times the callback will be called - during the file transfer. - - :type policy: :class:`boto.gs.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key - in GS. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum - of the file as the first element and the Base64-encoded version of - the plain checksum as the second element. This is the same format - returned by the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to - upload, it's silly to have to do it twice so this param, if present, - will be used as the MD5 values of the file. Otherwise, the checksum - will be computed. - - :type res_upload_handler: ResumableUploadHandler - :param res_upload_handler: If provided, this handler will perform the - upload. - """ - fp = open(filename, 'rb') - self.set_contents_from_file(fp, headers, replace, cb, num_cb, - policy, md5, res_upload_handler) - fp.close() diff --git a/backup/src/boto/gs/resumable_upload_handler.py b/backup/src/boto/gs/resumable_upload_handler.py deleted file mode 100644 index e8d5b03..0000000 --- a/backup/src/boto/gs/resumable_upload_handler.py +++ /dev/null @@ -1,526 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import cgi -import errno -import httplib -import os -import re -import socket -import time -import urlparse -import boto -from boto import config -from boto.connection import AWSAuthConnection -from boto.exception import InvalidUriError -from boto.exception import ResumableTransferDisposition -from boto.exception import ResumableUploadException - -""" -Handler for Google Storage resumable uploads. See -http://code.google.com/apis/storage/docs/developer-guide.html#resumable -for details. - -Resumable uploads will retry failed uploads, resuming at the byte -count completed by the last upload attempt. If too many retries happen with -no progress (per configurable num_retries param), the upload will be aborted. - -The caller can optionally specify a tracker_file_name param in the -ResumableUploadHandler constructor. If you do this, that file will -save the state needed to allow retrying later, in a separate process -(e.g., in a later run of gsutil). -""" - - -class ResumableUploadHandler(object): - - BUFFER_SIZE = 8192 - RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error, - socket.gaierror) - - # (start, end) response indicating server has nothing (upload protocol uses - # inclusive numbering). - SERVER_HAS_NOTHING = (0, -1) - - def __init__(self, tracker_file_name=None, num_retries=None): - """ - Constructor. Instantiate once for each uploaded file. - - :type tracker_file_name: string - :param tracker_file_name: optional file name to save tracker URI. - If supplied and the current process fails the upload, it can be - retried in a new process. If called with an existing file containing - a valid tracker URI, we'll resume the upload from this URI; else - we'll start a new resumable upload (and write the URI to this - tracker file). - - :type num_retries: int - :param num_retries: the number of times we'll re-try a resumable upload - making no progress. (Count resets every time we get progress, so - upload can span many more than this number of retries.) - """ - self.tracker_file_name = tracker_file_name - self.num_retries = num_retries - self.server_has_bytes = 0 # Byte count at last server check. - self.tracker_uri = None - if tracker_file_name: - self._load_tracker_uri_from_file() - # Save upload_start_point in instance state so caller can find how - # much was transferred by this ResumableUploadHandler (across retries). - self.upload_start_point = None - - def _load_tracker_uri_from_file(self): - f = None - try: - f = open(self.tracker_file_name, 'r') - uri = f.readline().strip() - self._set_tracker_uri(uri) - except IOError, e: - # Ignore non-existent file (happens first time an upload - # is attempted on a file), but warn user for other errors. - if e.errno != errno.ENOENT: - # Will restart because self.tracker_uri == None. - print('Couldn\'t read URI tracker file (%s): %s. Restarting ' - 'upload from scratch.' % - (self.tracker_file_name, e.strerror)) - except InvalidUriError, e: - # Warn user, but proceed (will restart because - # self.tracker_uri == None). - print('Invalid tracker URI (%s) found in URI tracker file ' - '(%s). Restarting upload from scratch.' % - (uri, self.tracker_file_name)) - finally: - if f: - f.close() - - def _save_tracker_uri_to_file(self): - """ - Saves URI to tracker file if one was passed to constructor. - """ - if not self.tracker_file_name: - return - f = None - try: - f = open(self.tracker_file_name, 'w') - f.write(self.tracker_uri) - except IOError, e: - raise ResumableUploadException( - 'Couldn\'t write URI tracker file (%s): %s.\nThis can happen' - 'if you\'re using an incorrectly configured upload tool\n' - '(e.g., gsutil configured to save tracker files to an ' - 'unwritable directory)' % - (self.tracker_file_name, e.strerror), - ResumableTransferDisposition.ABORT) - finally: - if f: - f.close() - - def _set_tracker_uri(self, uri): - """ - Called when we start a new resumable upload or get a new tracker - URI for the upload. Saves URI and resets upload state. - - Raises InvalidUriError if URI is syntactically invalid. - """ - parse_result = urlparse.urlparse(uri) - if (parse_result.scheme.lower() not in ['http', 'https'] or - not parse_result.netloc or not parse_result.query): - raise InvalidUriError('Invalid tracker URI (%s)' % uri) - qdict = cgi.parse_qs(parse_result.query) - if not qdict or not 'upload_id' in qdict: - raise InvalidUriError('Invalid tracker URI (%s)' % uri) - self.tracker_uri = uri - self.tracker_uri_host = parse_result.netloc - self.tracker_uri_path = '%s/?%s' % (parse_result.netloc, - parse_result.query) - self.server_has_bytes = 0 - - def get_tracker_uri(self): - """ - Returns upload tracker URI, or None if the upload has not yet started. - """ - return self.tracker_uri - - def _remove_tracker_file(self): - if (self.tracker_file_name and - os.path.exists(self.tracker_file_name)): - os.unlink(self.tracker_file_name) - - def _build_content_range_header(self, range_spec='*', length_spec='*'): - return 'bytes %s/%s' % (range_spec, length_spec) - - def _query_server_state(self, conn, file_length): - """ - Queries server to find out what bytes it currently has. - - Note that this method really just makes special case use of the - fact that the upload server always returns the current start/end - state whenever a PUT doesn't complete. - - Returns (server_start, server_end), where the values are inclusive. - For example, (0, 2) would mean that the server has bytes 0, 1, *and* 2. - - Raises ResumableUploadException if problem querying server. - """ - # Send an empty PUT so that server replies with this resumable - # transfer's state. - put_headers = {} - put_headers['Content-Range'] = ( - self._build_content_range_header('*', file_length)) - put_headers['Content-Length'] = '0' - resp = AWSAuthConnection.make_request(conn, 'PUT', - path=self.tracker_uri_path, - auth_path=self.tracker_uri_path, - headers=put_headers, - host=self.tracker_uri_host) - if resp.status == 200: - return (0, file_length) # Completed upload. - if resp.status != 308: - # This means the server didn't have any state for the given - # upload ID, which can happen (for example) if the caller saved - # the tracker URI to a file and then tried to restart the transfer - # after that upload ID has gone stale. In that case we need to - # start a new transfer (and the caller will then save the new - # tracker URI to the tracker file). - raise ResumableUploadException( - 'Got non-308 response (%s) from server state query' % - resp.status, ResumableTransferDisposition.START_OVER) - got_valid_response = False - range_spec = resp.getheader('range') - if range_spec: - # Parse 'bytes=-' range_spec. - m = re.search('bytes=(\d+)-(\d+)', range_spec) - if m: - server_start = long(m.group(1)) - server_end = long(m.group(2)) - got_valid_response = True - else: - # No Range header, which means the server does not yet have - # any bytes. Note that the Range header uses inclusive 'from' - # and 'to' values. Since Range 0-0 would mean that the server - # has byte 0, omitting the Range header is used to indicate that - # the server doesn't have any bytes. - return self.SERVER_HAS_NOTHING - if not got_valid_response: - raise ResumableUploadException( - 'Couldn\'t parse upload server state query response (%s)' % - str(resp.getheaders()), ResumableTransferDisposition.START_OVER) - if conn.debug >= 1: - print 'Server has: Range: %d - %d.' % (server_start, server_end) - return (server_start, server_end) - - def _start_new_resumable_upload(self, key, headers=None): - """ - Starts a new resumable upload. - - Raises ResumableUploadException if any errors occur. - """ - conn = key.bucket.connection - if conn.debug >= 1: - print 'Starting new resumable upload.' - self.server_has_bytes = 0 - - # Start a new resumable upload by sending a POST request with an - # empty body and the "X-Goog-Resumable: start" header. Include any - # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length - # (and raise an exception if they tried to pass one, since it's - # a semantic error to specify it at this point, and if we were to - # include one now it would cause the server to expect that many - # bytes; the POST doesn't include the actual file bytes We set - # the Content-Length in the subsequent PUT, based on the uploaded - # file size. - post_headers = {} - for k in headers: - if k.lower() == 'content-length': - raise ResumableUploadException( - 'Attempt to specify Content-Length header (disallowed)', - ResumableTransferDisposition.ABORT) - post_headers[k] = headers[k] - post_headers[conn.provider.resumable_upload_header] = 'start' - - resp = conn.make_request( - 'POST', key.bucket.name, key.name, post_headers) - # Get tracker URI from response 'Location' header. - body = resp.read() - # Check for '201 Created' response code. - if resp.status != 201: - raise ResumableUploadException( - 'Got status %d from attempt to start resumable upload' % - resp.status, ResumableTransferDisposition.WAIT_BEFORE_RETRY) - tracker_uri = resp.getheader('Location') - if not tracker_uri: - raise ResumableUploadException( - 'No resumable tracker URI found in resumable initiation ' - 'POST response (%s)' % body, - ResumableTransferDisposition.WAIT_BEFORE_RETRY) - self._set_tracker_uri(tracker_uri) - self._save_tracker_uri_to_file() - - def _upload_file_bytes(self, conn, http_conn, fp, file_length, - total_bytes_uploaded, cb, num_cb): - """ - Makes one attempt to upload file bytes, using an existing resumable - upload connection. - - Returns etag from server upon success. - - Raises ResumableUploadException if any problems occur. - """ - buf = fp.read(self.BUFFER_SIZE) - if cb: - if num_cb > 2: - cb_count = file_length / self.BUFFER_SIZE / (num_cb-2) - elif num_cb < 0: - cb_count = -1 - else: - cb_count = 0 - i = 0 - cb(total_bytes_uploaded, file_length) - - # Build resumable upload headers for the transfer. Don't send a - # Content-Range header if the file is 0 bytes long, because the - # resumable upload protocol uses an *inclusive* end-range (so, sending - # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). - put_headers = {} - if file_length: - range_header = self._build_content_range_header( - '%d-%d' % (total_bytes_uploaded, file_length - 1), - file_length) - put_headers['Content-Range'] = range_header - # Set Content-Length to the total bytes we'll send with this PUT. - put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) - http_request = AWSAuthConnection.build_base_http_request( - conn, 'PUT', path=self.tracker_uri_path, auth_path=None, - headers=put_headers, host=self.tracker_uri_host) - http_conn.putrequest('PUT', http_request.path) - for k in put_headers: - http_conn.putheader(k, put_headers[k]) - http_conn.endheaders() - - # Turn off debug on http connection so upload content isn't included - # in debug stream. - http_conn.set_debuglevel(0) - while buf: - http_conn.send(buf) - total_bytes_uploaded += len(buf) - if cb: - i += 1 - if i == cb_count or cb_count == -1: - cb(total_bytes_uploaded, file_length) - i = 0 - buf = fp.read(self.BUFFER_SIZE) - if cb: - cb(total_bytes_uploaded, file_length) - if total_bytes_uploaded != file_length: - raise ResumableUploadException('File changed during upload: EOF at ' - '%d bytes of %d byte file.' % - (total_bytes_uploaded, file_length), - ResumableTransferDisposition.ABORT) - resp = http_conn.getresponse() - body = resp.read() - # Restore http connection debug level. - http_conn.set_debuglevel(conn.debug) - - additional_note = '' - if resp.status == 200: - return resp.getheader('etag') # Success - # Retry status 503 errors after a delay. - elif resp.status == 503: - disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY - elif resp.status == 500: - disposition = ResumableTransferDisposition.ABORT - additional_note = ('This can happen if you attempt to upload a ' - 'different size file on a already partially ' - 'uploaded resumable upload') - else: - disposition = ResumableTransferDisposition.ABORT - raise ResumableUploadException('Got response code %d while attempting ' - 'upload (%s)%s' % - (resp.status, resp.reason, - additional_note), disposition) - - def _attempt_resumable_upload(self, key, fp, file_length, headers, cb, - num_cb): - """ - Attempts a resumable upload. - - Returns etag from server upon success. - - Raises ResumableUploadException if any problems occur. - """ - (server_start, server_end) = self.SERVER_HAS_NOTHING - conn = key.bucket.connection - if self.tracker_uri: - # Try to resume existing resumable upload. - try: - (server_start, server_end) = ( - self._query_server_state(conn, file_length)) - self.server_has_bytes = server_start - if conn.debug >= 1: - print 'Resuming transfer.' - except ResumableUploadException, e: - if conn.debug >= 1: - print 'Unable to resume transfer (%s).' % e.message - self._start_new_resumable_upload(key, headers) - else: - self._start_new_resumable_upload(key, headers) - - # upload_start_point allows the code that instantiated the - # ResumableUploadHandler to find out the point from which it started - # uploading (e.g., so it can correctly compute throughput). - if self.upload_start_point is None: - self.upload_start_point = server_end - - if server_end == file_length: - return # Done. - total_bytes_uploaded = server_end + 1 - fp.seek(total_bytes_uploaded) - conn = key.bucket.connection - - # Get a new HTTP connection (vs conn.get_http_connection(), which reuses - # pool connections) because httplib requires a new HTTP connection per - # transaction. (Without this, calling http_conn.getresponse() would get - # "ResponseNotReady".) - http_conn = conn.new_http_connection(self.tracker_uri_host, - conn.is_secure) - http_conn.set_debuglevel(conn.debug) - - # Make sure to close http_conn at end so if a local file read - # failure occurs partway through server will terminate current upload - # and can report that progress on next attempt. - try: - return self._upload_file_bytes(conn, http_conn, fp, file_length, - total_bytes_uploaded, cb, num_cb) - finally: - http_conn.close() - - def _check_final_md5(self, key, etag): - """ - Checks that etag from server agrees with md5 computed before upload. - This is important, since the upload could have spanned a number of - hours and multiple processes (e.g., gsutil runs), and the user could - change some of the file and not realize they have inconsistent data. - """ - if key.bucket.connection.debug >= 1: - print 'Checking md5 against etag.' - if key.md5 != etag.strip('"\''): - # Call key.open_read() before attempting to delete the - # (incorrect-content) key, so we perform that request on a - # different HTTP connection. This is neededb because httplib - # will return a "Response not ready" error if you try to perform - # a second transaction on the connection. - key.open_read() - key.close() - key.delete() - raise ResumableUploadException( - 'File changed during upload: md5 signature doesn\'t match etag ' - '(incorrect uploaded object deleted)', - ResumableTransferDisposition.ABORT) - - def send_file(self, key, fp, headers, cb=None, num_cb=10): - """ - Upload a file to a key into a bucket on GS, using GS resumable upload - protocol. - - :type key: :class:`boto.s3.key.Key` or subclass - :param key: The Key object to which data is to be uploaded - - :type fp: file-like object - :param fp: The file pointer to upload - - :type headers: dict - :param headers: The headers to pass along with the PUT request - - :type cb: function - :param cb: a callback function that will be called to report progress on - the upload. The callback should accept two integer parameters, the - first representing the number of bytes that have been successfully - transmitted to GS, and the second representing the total number of - bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter, this parameter determines the granularity of the callback - by defining the maximum number of times the callback will be called - during the file transfer. Providing a negative integer will cause - your callback to be called with each buffer read. - - Raises ResumableUploadException if a problem occurs during the transfer. - """ - - if not headers: - headers = {} - - fp.seek(0, os.SEEK_END) - file_length = fp.tell() - fp.seek(0) - debug = key.bucket.connection.debug - - # Use num-retries from constructor if one was provided; else check - # for a value specified in the boto config file; else default to 5. - if self.num_retries is None: - self.num_retries = config.getint('Boto', 'num_retries', 5) - progress_less_iterations = 0 - - while True: # Retry as long as we're making progress. - server_had_bytes_before_attempt = self.server_has_bytes - try: - etag = self._attempt_resumable_upload(key, fp, file_length, - headers, cb, num_cb) - # Upload succceded, so remove the tracker file (if have one). - self._remove_tracker_file() - self._check_final_md5(key, etag) - if debug >= 1: - print 'Resumable upload complete.' - return - except self.RETRYABLE_EXCEPTIONS, e: - if debug >= 1: - print('Caught exception (%s)' % e.__repr__()) - except ResumableUploadException, e: - if e.disposition == ResumableTransferDisposition.ABORT: - if debug >= 1: - print('Caught non-retryable ResumableUploadException ' - '(%s)' % e.message) - raise - else: - if debug >= 1: - print('Caught ResumableUploadException (%s) - will ' - 'retry' % e.message) - - # At this point we had a re-tryable failure; see if made progress. - if self.server_has_bytes > server_had_bytes_before_attempt: - progress_less_iterations = 0 - else: - progress_less_iterations += 1 - - if progress_less_iterations > self.num_retries: - # Don't retry any longer in the current process. - raise ResumableUploadException( - 'Too many resumable upload attempts failed without ' - 'progress. You might try this upload again later', - ResumableTransferDisposition.ABORT) - - sleep_time_secs = 2**progress_less_iterations - if debug >= 1: - print ('Got retryable failure (%d progress-less in a row).\n' - 'Sleeping %d seconds before re-trying' % - (progress_less_iterations, sleep_time_secs)) - time.sleep(sleep_time_secs) diff --git a/backup/src/boto/gs/user.py b/backup/src/boto/gs/user.py deleted file mode 100644 index 62f2cf5..0000000 --- a/backup/src/boto/gs/user.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -class User: - def __init__(self, parent=None, id='', name=''): - if parent: - parent.owner = self - self.type = None - self.id = id - self.name = name - - def __repr__(self): - return self.id - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Name': - self.name = value - elif name == 'ID': - self.id = value - else: - setattr(self, name, value) - - def to_xml(self, element_name='Owner'): - if self.type: - s = '<%s type="%s">' % (element_name, self.type) - else: - s = '<%s>' % element_name - s += '%s' % self.id - if self.name: - s += '%s' % self.name - s += '' % element_name - return s diff --git a/backup/src/boto/handler.py b/backup/src/boto/handler.py deleted file mode 100644 index 525f9c9..0000000 --- a/backup/src/boto/handler.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import xml.sax - -class XmlHandler(xml.sax.ContentHandler): - - def __init__(self, root_node, connection): - self.connection = connection - self.nodes = [('root', root_node)] - self.current_text = '' - - def startElement(self, name, attrs): - self.current_text = '' - new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) - if new_node != None: - self.nodes.append((name, new_node)) - - def endElement(self, name): - self.nodes[-1][1].endElement(name, self.current_text, self.connection) - if self.nodes[-1][0] == name: - self.nodes.pop() - self.current_text = '' - - def characters(self, content): - self.current_text += content - - diff --git a/backup/src/boto/iam/__init__.py b/backup/src/boto/iam/__init__.py deleted file mode 100644 index 498d736..0000000 --- a/backup/src/boto/iam/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# this is here for backward compatibility -# originally, the IAMConnection class was defined here -from connection import IAMConnection - - diff --git a/backup/src/boto/iam/connection.py b/backup/src/boto/iam/connection.py deleted file mode 100644 index 39ab704..0000000 --- a/backup/src/boto/iam/connection.py +++ /dev/null @@ -1,1006 +0,0 @@ -# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -import boto.jsonresponse -from boto.connection import AWSQueryConnection - -#boto.set_stream_logger('iam') - -class IAMConnection(AWSQueryConnection): - - APIVersion = '2010-05-08' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, host='iam.amazonaws.com', - debug=0, https_connection_factory=None, path='/'): - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, - proxy_port, proxy_user, proxy_pass, - host, debug, https_connection_factory, path) - - def _required_auth_capability(self): - return ['iam'] - - def get_response(self, action, params, path='/', parent=None, - verb='GET', list_marker='Set'): - """ - Utility method to handle calls to IAM and parsing of responses. - """ - if not parent: - parent = self - response = self.make_request(action, params, path, verb) - body = response.read() - boto.log.debug(body) - if response.status == 200: - e = boto.jsonresponse.Element(list_marker=list_marker) - h = boto.jsonresponse.XmlHandler(e, parent) - h.parse(body) - return e - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - # - # Group methods - # - - def get_all_groups(self, path_prefix='/', marker=None, max_items=None): - """ - List the groups that have the specified path prefix. - - :type path_prefix: string - :param path_prefix: If provided, only groups whose paths match - the provided prefix will be returned. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {} - if path_prefix: - params['PathPrefix'] = path_prefix - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListGroups', params, - list_marker='Groups') - - def get_group(self, group_name, marker=None, max_items=None): - """ - Return a list of users that are in the specified group. - - :type group_name: string - :param group_name: The name of the group whose information should - be returned. - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'GroupName' : group_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('GetGroup', params, list_marker='Users') - - def create_group(self, group_name, path='/'): - """ - Create a group. - - :type group_name: string - :param group_name: The name of the new group - - :type path: string - :param path: The path to the group (Optional). Defaults to /. - - """ - params = {'GroupName' : group_name, - 'Path' : path} - return self.get_response('CreateGroup', params) - - def delete_group(self, group_name): - """ - Delete a group. The group must not contain any Users or - have any attached policies - - :type group_name: string - :param group_name: The name of the group to delete. - - """ - params = {'GroupName' : group_name} - return self.get_response('DeleteGroup', params) - - def update_group(self, group_name, new_group_name=None, new_path=None): - """ - Updates name and/or path of the specified group. - - :type group_name: string - :param group_name: The name of the new group - - :type new_group_name: string - :param new_group_name: If provided, the name of the group will be - changed to this name. - - :type new_path: string - :param new_path: If provided, the path of the group will be - changed to this path. - - """ - params = {'GroupName' : group_name} - if new_group_name: - params['NewGroupName'] = new_group_name - if new_path: - params['NewPath'] = new_path - return self.get_response('UpdateGroup', params) - - def add_user_to_group(self, group_name, user_name): - """ - Add a user to a group - - :type group_name: string - :param group_name: The name of the new group - - :type user_name: string - :param user_name: The to be added to the group. - - """ - params = {'GroupName' : group_name, - 'UserName' : user_name} - return self.get_response('AddUserToGroup', params) - - def remove_user_from_group(self, group_name, user_name): - """ - Remove a user from a group. - - :type group_name: string - :param group_name: The name of the new group - - :type user_name: string - :param user_name: The user to remove from the group. - - """ - params = {'GroupName' : group_name, - 'UserName' : user_name} - return self.get_response('RemoveUserFromGroup', params) - - def put_group_policy(self, group_name, policy_name, policy_json): - """ - Adds or updates the specified policy document for the specified group. - - :type group_name: string - :param group_name: The name of the group the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to get. - - :type policy_json: string - :param policy_json: The policy document. - - """ - params = {'GroupName' : group_name, - 'PolicyName' : policy_name, - 'PolicyDocument' : policy_json} - return self.get_response('PutGroupPolicy', params, verb='POST') - - def get_all_group_policies(self, group_name, marker=None, max_items=None): - """ - List the names of the policies associated with the specified group. - - :type group_name: string - :param group_name: The name of the group the policy is associated with. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'GroupName' : group_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListGroupPolicies', params, - list_marker='PolicyNames') - - def get_group_policy(self, group_name, policy_name): - """ - Retrieves the specified policy document for the specified group. - - :type group_name: string - :param group_name: The name of the group the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to get. - - """ - params = {'GroupName' : group_name, - 'PolicyName' : policy_name} - return self.get_response('GetGroupPolicy', params, verb='POST') - - def delete_group_policy(self, group_name, policy_name): - """ - Deletes the specified policy document for the specified group. - - :type group_name: string - :param group_name: The name of the group the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to delete. - - """ - params = {'GroupName' : group_name, - 'PolicyName' : policy_name} - return self.get_response('DeleteGroupPolicy', params, verb='POST') - - def get_all_users(self, path_prefix='/', marker=None, max_items=None): - """ - List the users that have the specified path prefix. - - :type path_prefix: string - :param path_prefix: If provided, only users whose paths match - the provided prefix will be returned. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'PathPrefix' : path_prefix} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListUsers', params, list_marker='Users') - - # - # User methods - # - - def create_user(self, user_name, path='/'): - """ - Create a user. - - :type user_name: string - :param user_name: The name of the new user - - :type path: string - :param path: The path in which the user will be created. - Defaults to /. - - """ - params = {'UserName' : user_name, - 'Path' : path} - return self.get_response('CreateUser', params) - - def delete_user(self, user_name): - """ - Delete a user including the user's path, GUID and ARN. - - If the user_name is not specified, the user_name is determined - implicitly based on the AWS Access Key ID used to sign the request. - - :type user_name: string - :param user_name: The name of the user to delete. - - """ - params = {'UserName' : user_name} - return self.get_response('DeleteUser', params) - - def get_user(self, user_name=None): - """ - Retrieve information about the specified user. - - If the user_name is not specified, the user_name is determined - implicitly based on the AWS Access Key ID used to sign the request. - - :type user_name: string - :param user_name: The name of the user to delete. - If not specified, defaults to user making - request. - - """ - params = {} - if user_name: - params['UserName'] = user_name - return self.get_response('GetUser', params) - - def update_user(self, user_name, new_user_name=None, new_path=None): - """ - Updates name and/or path of the specified user. - - :type user_name: string - :param user_name: The name of the user - - :type new_user_name: string - :param new_user_name: If provided, the username of the user will be - changed to this username. - - :type new_path: string - :param new_path: If provided, the path of the user will be - changed to this path. - - """ - params = {'UserName' : user_name} - if new_user_name: - params['NewUserName'] = new_user_name - if new_path: - params['NewPath'] = new_path - return self.get_response('UpdateUser', params) - - def get_all_user_policies(self, user_name, marker=None, max_items=None): - """ - List the names of the policies associated with the specified user. - - :type user_name: string - :param user_name: The name of the user the policy is associated with. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'UserName' : user_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListUserPolicies', params, - list_marker='PolicyNames') - - def put_user_policy(self, user_name, policy_name, policy_json): - """ - Adds or updates the specified policy document for the specified user. - - :type user_name: string - :param user_name: The name of the user the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to get. - - :type policy_json: string - :param policy_json: The policy document. - - """ - params = {'UserName' : user_name, - 'PolicyName' : policy_name, - 'PolicyDocument' : policy_json} - return self.get_response('PutUserPolicy', params, verb='POST') - - def get_user_policy(self, user_name, policy_name): - """ - Retrieves the specified policy document for the specified user. - - :type user_name: string - :param user_name: The name of the user the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to get. - - """ - params = {'UserName' : user_name, - 'PolicyName' : policy_name} - return self.get_response('GetUserPolicy', params, verb='POST') - - def delete_user_policy(self, user_name, policy_name): - """ - Deletes the specified policy document for the specified user. - - :type user_name: string - :param user_name: The name of the user the policy is associated with. - - :type policy_name: string - :param policy_name: The policy document to delete. - - """ - params = {'UserName' : user_name, - 'PolicyName' : policy_name} - return self.get_response('DeleteUserPolicy', params, verb='POST') - - def get_groups_for_user(self, user_name, marker=None, max_items=None): - """ - List the groups that a specified user belongs to. - - :type user_name: string - :param user_name: The name of the user to list groups for. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'UserName' : user_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListGroupsForUser', params, - list_marker='Groups') - - # - # Access Keys - # - - def get_all_access_keys(self, user_name, marker=None, max_items=None): - """ - Get all access keys associated with an account. - - :type user_name: string - :param user_name: The username of the new user - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - """ - params = {'UserName' : user_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListAccessKeys', params, - list_marker='AccessKeyMetadata') - - def create_access_key(self, user_name=None): - """ - Create a new AWS Secret Access Key and corresponding AWS Access Key ID - for the specified user. The default status for new keys is Active - - If the user_name is not specified, the user_name is determined - implicitly based on the AWS Access Key ID used to sign the request. - - :type user_name: string - :param user_name: The username of the new user - - """ - params = {'UserName' : user_name} - return self.get_response('CreateAccessKey', params) - - def update_access_key(self, access_key_id, status, user_name=None): - """ - Changes the status of the specified access key from Active to Inactive - or vice versa. This action can be used to disable a user's key as - part of a key rotation workflow. - - If the user_name is not specified, the user_name is determined - implicitly based on the AWS Access Key ID used to sign the request. - - :type access_key_id: string - :param access_key_id: The ID of the access key. - - :type status: string - :param status: Either Active or Inactive. - - :type user_name: string - :param user_name: The username of user (optional). - - """ - params = {'AccessKeyId' : access_key_id, - 'Status' : status} - if user_name: - params['UserName'] = user_name - return self.get_response('UpdateAccessKey', params) - - def delete_access_key(self, access_key_id, user_name=None): - """ - Delete an access key associated with a user. - - If the user_name is not specified, it is determined implicitly based - on the AWS Access Key ID used to sign the request. - - :type access_key_id: string - :param access_key_id: The ID of the access key to be deleted. - - :type user_name: string - :param user_name: The username of the new user - - """ - params = {'AccessKeyId' : access_key_id} - if user_name: - params['UserName'] = user_name - return self.get_response('DeleteAccessKey', params) - - # - # Signing Certificates - # - - def get_all_signing_certs(self, marker=None, max_items=None, - user_name=None): - """ - Get all signing certificates associated with an account. - - If the user_name is not specified, it is determined implicitly based - on the AWS Access Key ID used to sign the request. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - - :type user_name: string - :param user_name: The username of the user - - """ - params = {} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - if user_name: - params['UserName'] = user_name - return self.get_response('ListSigningCertificates', - params, list_marker='Certificates') - - def update_signing_cert(self, cert_id, status, user_name=None): - """ - Change the status of the specified signing certificate from - Active to Inactive or vice versa. - - If the user_name is not specified, it is determined implicitly based - on the AWS Access Key ID used to sign the request. - - :type cert_id: string - :param cert_id: The ID of the signing certificate - - :type status: string - :param status: Either Active or Inactive. - - :type user_name: string - :param user_name: The username of the user - """ - params = {'CertificateId' : cert_id, - 'Status' : status} - if user_name: - params['UserName'] = user_name - return self.get_response('UpdateSigningCertificate', params) - - def upload_signing_cert(self, cert_body, user_name=None): - """ - Uploads an X.509 signing certificate and associates it with - the specified user. - - If the user_name is not specified, it is determined implicitly based - on the AWS Access Key ID used to sign the request. - - :type cert_body: string - :param cert_body: The body of the signing certificate. - - :type user_name: string - :param user_name: The username of the new user - - """ - params = {'CertificateBody' : cert_body} - if user_name: - params['UserName'] = user_name - return self.get_response('UploadSigningCertificate', params, - verb='POST') - - def delete_signing_cert(self, cert_id, user_name=None): - """ - Delete a signing certificate associated with a user. - - If the user_name is not specified, it is determined implicitly based - on the AWS Access Key ID used to sign the request. - - :type user_name: string - :param user_name: The username of the new user - - :type cert_id: string - :param cert_id: The ID of the certificate. - - """ - params = {'CertificateId' : cert_id} - if user_name: - params['UserName'] = user_name - return self.get_response('DeleteSigningCertificate', params) - - # - # Server Certificates - # - - def get_all_server_certs(self, path_prefix='/', - marker=None, max_items=None): - """ - Lists the server certificates that have the specified path prefix. - If none exist, the action returns an empty list. - - :type path_prefix: string - :param path_prefix: If provided, only certificates whose paths match - the provided prefix will be returned. - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - - """ - params = {} - if path_prefix: - params['PathPrefix'] = path_prefix - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListServerCertificates', - params, - list_marker='ServerCertificateMetadataList') - - def update_server_cert(self, cert_name, new_cert_name=None, - new_path=None): - """ - Updates the name and/or the path of the specified server certificate. - - :type cert_name: string - :param cert_name: The name of the server certificate that you want - to update. - - :type new_cert_name: string - :param new_cert_name: The new name for the server certificate. - Include this only if you are updating the - server certificate's name. - - :type new_path: string - :param new_path: If provided, the path of the certificate will be - changed to this path. - """ - params = {'ServerCertificateName' : cert_name} - if new_cert_name: - params['NewServerCertificateName'] = new_cert_name - if new_path: - params['NewPath'] = new_path - return self.get_response('UpdateServerCertificate', params) - - def upload_server_cert(self, cert_name, cert_body, private_key, - cert_chain=None, path=None): - """ - Uploads a server certificate entity for the AWS Account. - The server certificate entity includes a public key certificate, - a private key, and an optional certificate chain, which should - all be PEM-encoded. - - :type cert_name: string - :param cert_name: The name for the server certificate. Do not - include the path in this value. - - :type cert_body: string - :param cert_body: The contents of the public key certificate - in PEM-encoded format. - - :type private_key: string - :param private_key: The contents of the private key in - PEM-encoded format. - - :type cert_chain: string - :param cert_chain: The contents of the certificate chain. This - is typically a concatenation of the PEM-encoded - public key certificates of the chain. - - :type path: string - :param path: The path for the server certificate. - - """ - params = {'ServerCertificateName' : cert_name, - 'CertificateBody' : cert_body, - 'PrivateKey' : private_key} - if cert_chain: - params['CertificateChain'] = cert_chain - if path: - params['Path'] = path - return self.get_response('UploadServerCertificate', params, - verb='POST') - - def get_server_certificate(self, cert_name): - """ - Retrieves information about the specified server certificate. - - :type cert_name: string - :param cert_name: The name of the server certificate you want - to retrieve information about. - - """ - params = {'ServerCertificateName' : cert_name} - return self.get_response('GetServerCertificate', params) - - def delete_server_cert(self, cert_name): - """ - Delete the specified server certificate. - - :type cert_name: string - :param cert_name: The name of the server certificate you want - to delete. - - """ - params = {'ServerCertificateName' : cert_name} - return self.get_response('DeleteServerCertificate', params) - - # - # MFA Devices - # - - def get_all_mfa_devices(self, user_name, marker=None, max_items=None): - """ - Get all MFA devices associated with an account. - - :type user_name: string - :param user_name: The username of the user - - :type marker: string - :param marker: Use this only when paginating results and only in - follow-up request after you've received a response - where the results are truncated. Set this to the - value of the Marker element in the response you - just received. - - :type max_items: int - :param max_items: Use this only when paginating results to indicate - the maximum number of groups you want in the - response. - - """ - params = {'UserName' : user_name} - if marker: - params['Marker'] = marker - if max_items: - params['MaxItems'] = max_items - return self.get_response('ListMFADevices', - params, list_marker='MFADevices') - - def enable_mfa_device(self, user_name, serial_number, - auth_code_1, auth_code_2): - """ - Enables the specified MFA device and associates it with the - specified user. - - :type user_name: string - :param user_name: The username of the user - - :type serial_number: string - :param seriasl_number: The serial number which uniquely identifies - the MFA device. - - :type auth_code_1: string - :param auth_code_1: An authentication code emitted by the device. - - :type auth_code_2: string - :param auth_code_2: A subsequent authentication code emitted - by the device. - - """ - params = {'UserName' : user_name, - 'SerialNumber' : serial_number, - 'AuthenticationCode1' : auth_code_1, - 'AuthenticationCode2' : auth_code_2} - return self.get_response('EnableMFADevice', params) - - def deactivate_mfa_device(self, user_name, serial_number): - """ - Deactivates the specified MFA device and removes it from - association with the user. - - :type user_name: string - :param user_name: The username of the user - - :type serial_number: string - :param seriasl_number: The serial number which uniquely identifies - the MFA device. - - """ - params = {'UserName' : user_name, - 'SerialNumber' : serial_number} - return self.get_response('DeactivateMFADevice', params) - - def resync_mfa_device(self, user_name, serial_number, - auth_code_1, auth_code_2): - """ - Syncronizes the specified MFA device with the AWS servers. - - :type user_name: string - :param user_name: The username of the user - - :type serial_number: string - :param seriasl_number: The serial number which uniquely identifies - the MFA device. - - :type auth_code_1: string - :param auth_code_1: An authentication code emitted by the device. - - :type auth_code_2: string - :param auth_code_2: A subsequent authentication code emitted - by the device. - - """ - params = {'UserName' : user_name, - 'SerialNumber' : serial_number, - 'AuthenticationCode1' : auth_code_1, - 'AuthenticationCode2' : auth_code_2} - return self.get_response('ResyncMFADevice', params) - - # - # Login Profiles - # - - def create_login_profile(self, user_name, password): - """ - Creates a login profile for the specified user, give the user the - ability to access AWS services and the AWS Management Console. - - :type user_name: string - :param user_name: The name of the new user - - :type password: string - :param password: The new password for the user - - """ - params = {'UserName' : user_name, - 'Password' : password} - return self.get_response('CreateLoginProfile', params) - - def delete_login_profile(self, user_name): - """ - Deletes the login profile associated with the specified user. - - :type user_name: string - :param user_name: The name of the user to delete. - - """ - params = {'UserName' : user_name} - return self.get_response('DeleteLoginProfile', params) - - def update_login_profile(self, user_name, password): - """ - Resets the password associated with the user's login profile. - - :type user_name: string - :param user_name: The name of the user - - :type password: string - :param password: The new password for the user - - """ - params = {'UserName' : user_name, - 'Password' : password} - return self.get_response('UpdateLoginProfile', params) - - def create_account_alias(self, alias): - """ - Creates a new alias for the AWS account. - - For more information on account id aliases, please see - http://goo.gl/ToB7G - - :type alias: string - :param alias: The alias to attach to the account. - """ - params = {'AccountAlias': alias} - return self.get_response('CreateAccountAlias', params) - - def delete_account_alias(self, alias): - """ - Deletes an alias for the AWS account. - - For more information on account id aliases, please see - http://goo.gl/ToB7G - - :type alias: string - :param alias: The alias to remove from the account. - """ - params = {'AccountAlias': alias} - return self.get_response('DeleteAccountAlias', params) - - def get_account_alias(self): - """ - Get the alias for the current account. - - This is referred to in the docs as list_account_aliases, - but it seems you can only have one account alias currently. - - For more information on account id aliases, please see - http://goo.gl/ToB7G - """ - r = self.get_response('ListAccountAliases', {}) - response = r.get('ListAccountAliasesResponse') - result = response.get('ListAccountAliasesResult') - aliases = result.get('AccountAliases') - return aliases.get('member', None) - - def get_signin_url(self, service='ec2'): - """ - Get the URL where IAM users can use their login profile to sign in - to this account's console. - - :type service: string - :param service: Default service to go to in the console. - """ - alias = self.get_account_alias() - if not alias: - raise Exception('No alias associated with this account. Please use iam.create_account_alias() first.') - - return "https://%s.signin.aws.amazon.com/console/%s" % (alias, service) diff --git a/backup/src/boto/jsonresponse.py b/backup/src/boto/jsonresponse.py deleted file mode 100644 index beb50ce..0000000 --- a/backup/src/boto/jsonresponse.py +++ /dev/null @@ -1,143 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import xml.sax -import utils - -class XmlHandler(xml.sax.ContentHandler): - - def __init__(self, root_node, connection): - self.connection = connection - self.nodes = [('root', root_node)] - self.current_text = '' - - def startElement(self, name, attrs): - self.current_text = '' - t = self.nodes[-1][1].startElement(name, attrs, self.connection) - if t != None: - if isinstance(t, tuple): - self.nodes.append(t) - else: - self.nodes.append((name, t)) - - def endElement(self, name): - self.nodes[-1][1].endElement(name, self.current_text, self.connection) - if self.nodes[-1][0] == name: - self.nodes.pop() - self.current_text = '' - - def characters(self, content): - self.current_text += content - - def parse(self, s): - xml.sax.parseString(s, self) - -class Element(dict): - - def __init__(self, connection=None, element_name=None, - stack=None, parent=None, list_marker=('Set',), - item_marker=('member', 'item')): - dict.__init__(self) - self.connection = connection - self.element_name = element_name - self.list_marker = utils.mklist(list_marker) - self.item_marker = utils.mklist(item_marker) - if stack is None: - self.stack = [] - else: - self.stack = stack - self.parent = parent - - def __getattr__(self, key): - if key in self: - return self[key] - for k in self: - e = self[k] - if isinstance(e, Element): - try: - return getattr(e, key) - except AttributeError: - pass - raise AttributeError - - def startElement(self, name, attrs, connection): - self.stack.append(name) - for lm in self.list_marker: - if name.endswith(lm): - l = ListElement(self.connection, name, self.list_marker, - self.item_marker) - self[name] = l - return l - if len(self.stack) > 0: - element_name = self.stack[-1] - e = Element(self.connection, element_name, self.stack, self, - self.list_marker, self.item_marker) - self[element_name] = e - return (element_name, e) - else: - return None - - def endElement(self, name, value, connection): - if len(self.stack) > 0: - self.stack.pop() - value = value.strip() - if value: - if isinstance(self.parent, Element): - self.parent[name] = value - elif isinstance(self.parent, ListElement): - self.parent.append(value) - -class ListElement(list): - - def __init__(self, connection=None, element_name=None, - list_marker=['Set'], item_marker=('member', 'item')): - list.__init__(self) - self.connection = connection - self.element_name = element_name - self.list_marker = list_marker - self.item_marker = item_marker - - def startElement(self, name, attrs, connection): - for lm in self.list_marker: - if name.endswith(lm): - l = ListElement(self.connection, name, self.item_marker) - setattr(self, name, l) - return l - if name in self.item_marker: - e = Element(self.connection, name, parent=self) - self.append(e) - return e - else: - return None - - def endElement(self, name, value, connection): - if name == self.element_name: - if len(self) > 0: - empty = [] - for e in self: - if isinstance(e, Element): - if len(e) == 0: - empty.append(e) - for e in empty: - self.remove(e) - else: - setattr(self, name, value) diff --git a/backup/src/boto/manage/__init__.py b/backup/src/boto/manage/__init__.py deleted file mode 100644 index 49d029b..0000000 --- a/backup/src/boto/manage/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - - diff --git a/backup/src/boto/manage/cmdshell.py b/backup/src/boto/manage/cmdshell.py deleted file mode 100644 index cbd2e60..0000000 --- a/backup/src/boto/manage/cmdshell.py +++ /dev/null @@ -1,174 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.mashups.interactive import interactive_shell -import boto -import os -import time -import shutil -import StringIO -import paramiko -import socket -import subprocess - - -class SSHClient(object): - - def __init__(self, server, - host_key_file='~/.ssh/known_hosts', - uname='root', ssh_pwd=None): - self.server = server - self.host_key_file = host_key_file - self.uname = uname - self._pkey = paramiko.RSAKey.from_private_key_file(server.ssh_key_file, - password=ssh_pwd) - self._ssh_client = paramiko.SSHClient() - self._ssh_client.load_system_host_keys() - self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) - self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self.connect() - - def connect(self): - retry = 0 - while retry < 5: - try: - self._ssh_client.connect(self.server.hostname, - username=self.uname, - pkey=self._pkey) - return - except socket.error, (value,message): - if value == 61 or value == 111: - print 'SSH Connection refused, will retry in 5 seconds' - time.sleep(5) - retry += 1 - else: - raise - except paramiko.BadHostKeyException: - print "%s has an entry in ~/.ssh/known_hosts and it doesn't match" % self.server.hostname - print 'Edit that file to remove the entry and then hit return to try again' - raw_input('Hit Enter when ready') - retry += 1 - except EOFError: - print 'Unexpected Error from SSH Connection, retry in 5 seconds' - time.sleep(5) - retry += 1 - print 'Could not establish SSH connection' - - def get_file(self, src, dst): - sftp_client = self._ssh_client.open_sftp() - sftp_client.get(src, dst) - - def put_file(self, src, dst): - sftp_client = self._ssh_client.open_sftp() - sftp_client.put(src, dst) - - def listdir(self, path): - sftp_client = self._ssh_client.open_sftp() - return sftp_client.listdir(path) - - def open_sftp(self): - return self._ssh_client.open_sftp() - - def isdir(self, path): - status = self.run('[ -d %s ] || echo "FALSE"' % path) - if status[1].startswith('FALSE'): - return 0 - return 1 - - def exists(self, path): - status = self.run('[ -a %s ] || echo "FALSE"' % path) - if status[1].startswith('FALSE'): - return 0 - return 1 - - def shell(self): - channel = self._ssh_client.invoke_shell() - interactive_shell(channel) - - def run(self, command): - boto.log.info('running:%s on %s' % (command, self.server.instance_id)) - log_fp = StringIO.StringIO() - status = 0 - try: - t = self._ssh_client.exec_command(command) - except paramiko.SSHException: - status = 1 - log_fp.write(t[1].read()) - log_fp.write(t[2].read()) - t[0].close() - t[1].close() - t[2].close() - boto.log.info('output: %s' % log_fp.getvalue()) - return (status, log_fp.getvalue()) - - def close(self): - transport = self._ssh_client.get_transport() - transport.close() - self.server.reset_cmdshell() - -class LocalClient(object): - - def __init__(self, server, host_key_file=None, uname='root'): - self.server = server - self.host_key_file = host_key_file - self.uname = uname - - def get_file(self, src, dst): - shutil.copyfile(src, dst) - - def put_file(self, src, dst): - shutil.copyfile(src, dst) - - def listdir(self, path): - return os.listdir(path) - - def isdir(self, path): - return os.path.isdir(path) - - def exists(self, path): - return os.path.exists(path) - - def shell(self): - raise NotImplementedError, 'shell not supported with LocalClient' - - def run(self): - boto.log.info('running:%s' % self.command) - log_fp = StringIO.StringIO() - process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - while process.poll() == None: - time.sleep(1) - t = process.communicate() - log_fp.write(t[0]) - log_fp.write(t[1]) - boto.log.info(log_fp.getvalue()) - boto.log.info('output: %s' % log_fp.getvalue()) - return (process.returncode, log_fp.getvalue()) - - def close(self): - pass - -def start(server): - instance_id = boto.config.get('Instance', 'instance-id', None) - if instance_id == server.instance_id: - return LocalClient(server) - else: - return SSHClient(server) diff --git a/backup/src/boto/manage/propget.py b/backup/src/boto/manage/propget.py deleted file mode 100644 index 45b2ff2..0000000 --- a/backup/src/boto/manage/propget.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -def get(prop, choices=None): - prompt = prop.verbose_name - if not prompt: - prompt = prop.name - if choices: - if callable(choices): - choices = choices() - else: - choices = prop.get_choices() - valid = False - while not valid: - if choices: - min = 1 - max = len(choices) - for i in range(min, max+1): - value = choices[i-1] - if isinstance(value, tuple): - value = value[0] - print '[%d] %s' % (i, value) - value = raw_input('%s [%d-%d]: ' % (prompt, min, max)) - try: - int_value = int(value) - value = choices[int_value-1] - if isinstance(value, tuple): - value = value[1] - valid = True - except ValueError: - print '%s is not a valid choice' % value - except IndexError: - print '%s is not within the range[%d-%d]' % (min, max) - else: - value = raw_input('%s: ' % prompt) - try: - value = prop.validate(value) - if prop.empty(value) and prop.required: - print 'A value is required' - else: - valid = True - except: - print 'Invalid value: %s' % value - return value - diff --git a/backup/src/boto/manage/server.py b/backup/src/boto/manage/server.py deleted file mode 100644 index 3c7a303..0000000 --- a/backup/src/boto/manage/server.py +++ /dev/null @@ -1,556 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -High-level abstraction of an EC2 server -""" -from __future__ import with_statement -import boto.ec2 -from boto.mashups.iobject import IObject -from boto.pyami.config import BotoConfigPath, Config -from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty, CalculatedProperty -from boto.manage import propget -from boto.ec2.zone import Zone -from boto.ec2.keypair import KeyPair -import os, time, StringIO -from contextlib import closing -from boto.exception import EC2ResponseError - -InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', - 'c1.medium', 'c1.xlarge', - 'm2.2xlarge', 'm2.4xlarge'] - -class Bundler(object): - - def __init__(self, server, uname='root'): - from boto.manage.cmdshell import SSHClient - self.server = server - self.uname = uname - self.ssh_client = SSHClient(server, uname=uname) - - def copy_x509(self, key_file, cert_file): - print '\tcopying cert and pk over to /mnt directory on server' - self.ssh_client.open_sftp() - path, name = os.path.split(key_file) - self.remote_key_file = '/mnt/%s' % name - self.ssh_client.put_file(key_file, self.remote_key_file) - path, name = os.path.split(cert_file) - self.remote_cert_file = '/mnt/%s' % name - self.ssh_client.put_file(cert_file, self.remote_cert_file) - print '...complete!' - - def bundle_image(self, prefix, size, ssh_key): - command = "" - if self.uname != 'root': - command = "sudo " - command += 'ec2-bundle-vol ' - command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file) - command += '-u %s ' % self.server._reservation.owner_id - command += '-p %s ' % prefix - command += '-s %d ' % size - command += '-d /mnt ' - if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium': - command += '-r i386' - else: - command += '-r x86_64' - return command - - def upload_bundle(self, bucket, prefix, ssh_key): - command = "" - if self.uname != 'root': - command = "sudo " - command += 'ec2-upload-bundle ' - command += '-m /mnt/%s.manifest.xml ' % prefix - command += '-b %s ' % bucket - command += '-a %s ' % self.server.ec2.aws_access_key_id - command += '-s %s ' % self.server.ec2.aws_secret_access_key - return command - - def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, - size=None, ssh_key=None, fp=None, clear_history=True): - iobject = IObject() - if not bucket: - bucket = iobject.get_string('Name of S3 bucket') - if not prefix: - prefix = iobject.get_string('Prefix for AMI file') - if not key_file: - key_file = iobject.get_filename('Path to RSA private key file') - if not cert_file: - cert_file = iobject.get_filename('Path to RSA public cert file') - if not size: - size = iobject.get_int('Size (in MB) of bundled image') - if not ssh_key: - ssh_key = self.server.get_ssh_key_file() - self.copy_x509(key_file, cert_file) - if not fp: - fp = StringIO.StringIO() - fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath) - fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ') - if clear_history: - fp.write('history -c; ') - fp.write(self.bundle_image(prefix, size, ssh_key)) - fp.write('; ') - fp.write(self.upload_bundle(bucket, prefix, ssh_key)) - fp.write('; ') - fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath) - fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys') - command = fp.getvalue() - print 'running the following command on the remote server:' - print command - t = self.ssh_client.run(command) - print '\t%s' % t[0] - print '\t%s' % t[1] - print '...complete!' - print 'registering image...' - self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix)) - return self.image_id - -class CommandLineGetter(object): - - def get_ami_list(self): - my_amis = [] - for ami in self.ec2.get_all_images(): - # hack alert, need a better way to do this! - if ami.location.find('pyami') >= 0: - my_amis.append((ami.location, ami)) - return my_amis - - def get_region(self, params): - region = params.get('region', None) - if isinstance(region, str) or isinstance(region, unicode): - region = boto.ec2.get_region(region) - params['region'] = region - if not region: - prop = self.cls.find_property('region_name') - params['region'] = propget.get(prop, choices=boto.ec2.regions) - self.ec2 = params['region'].connect() - - def get_name(self, params): - if not params.get('name', None): - prop = self.cls.find_property('name') - params['name'] = propget.get(prop) - - def get_description(self, params): - if not params.get('description', None): - prop = self.cls.find_property('description') - params['description'] = propget.get(prop) - - def get_instance_type(self, params): - if not params.get('instance_type', None): - prop = StringProperty(name='instance_type', verbose_name='Instance Type', - choices=InstanceTypes) - params['instance_type'] = propget.get(prop) - - def get_quantity(self, params): - if not params.get('quantity', None): - prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') - params['quantity'] = propget.get(prop) - - def get_zone(self, params): - if not params.get('zone', None): - prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', - choices=self.ec2.get_all_zones) - params['zone'] = propget.get(prop) - - def get_ami_id(self, params): - valid = False - while not valid: - ami = params.get('ami', None) - if not ami: - prop = StringProperty(name='ami', verbose_name='AMI') - ami = propget.get(prop) - try: - rs = self.ec2.get_all_images([ami]) - if len(rs) == 1: - valid = True - params['ami'] = rs[0] - except EC2ResponseError: - pass - - def get_group(self, params): - group = params.get('group', None) - if isinstance(group, str) or isinstance(group, unicode): - group_list = self.ec2.get_all_security_groups() - for g in group_list: - if g.name == group: - group = g - params['group'] = g - if not group: - prop = StringProperty(name='group', verbose_name='EC2 Security Group', - choices=self.ec2.get_all_security_groups) - params['group'] = propget.get(prop) - - def get_key(self, params): - keypair = params.get('keypair', None) - if isinstance(keypair, str) or isinstance(keypair, unicode): - key_list = self.ec2.get_all_key_pairs() - for k in key_list: - if k.name == keypair: - keypair = k.name - params['keypair'] = k.name - if not keypair: - prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair', - choices=self.ec2.get_all_key_pairs) - params['keypair'] = propget.get(prop).name - - def get(self, cls, params): - self.cls = cls - self.get_region(params) - self.ec2 = params['region'].connect() - self.get_name(params) - self.get_description(params) - self.get_instance_type(params) - self.get_zone(params) - self.get_quantity(params) - self.get_ami_id(params) - self.get_group(params) - self.get_key(params) - -class Server(Model): - - # - # The properties of this object consists of real properties for data that - # is not already stored in EC2 somewhere (e.g. name, description) plus - # calculated properties for all of the properties that are already in - # EC2 (e.g. hostname, security groups, etc.) - # - name = StringProperty(unique=True, verbose_name="Name") - description = StringProperty(verbose_name="Description") - region_name = StringProperty(verbose_name="EC2 Region Name") - instance_id = StringProperty(verbose_name="EC2 Instance ID") - elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address") - production = BooleanProperty(verbose_name="Is This Server Production", default=False) - ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True) - zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True) - hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True) - private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True) - groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True) - security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True) - key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True) - instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True) - status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True) - launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True) - console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True) - - packages = [] - plugins = [] - - @classmethod - def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key): - if not cfg.has_section('Credentials'): - cfg.add_section('Credentials') - cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id) - cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key) - if not cfg.has_section('DB_Server'): - cfg.add_section('DB_Server') - cfg.set('DB_Server', 'db_type', 'SimpleDB') - cfg.set('DB_Server', 'db_name', cls._manager.domain.name) - - @classmethod - def create(cls, config_file=None, logical_volume = None, cfg = None, **params): - """ - Create a new instance based on the specified configuration file or the specified - configuration and the passed in parameters. - - If the config_file argument is not None, the configuration is read from there. - Otherwise, the cfg argument is used. - - The config file may include other config files with a #import reference. The included - config files must reside in the same directory as the specified file. - - The logical_volume argument, if supplied, will be used to get the current physical - volume ID and use that as an override of the value specified in the config file. This - may be useful for debugging purposes when you want to debug with a production config - file but a test Volume. - - The dictionary argument may be used to override any EC2 configuration values in the - config file. - """ - if config_file: - cfg = Config(path=config_file) - if cfg.has_section('EC2'): - # include any EC2 configuration values that aren't specified in params: - for option in cfg.options('EC2'): - if option not in params: - params[option] = cfg.get('EC2', option) - getter = CommandLineGetter() - getter.get(cls, params) - region = params.get('region') - ec2 = region.connect() - cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key) - ami = params.get('ami') - kp = params.get('keypair') - group = params.get('group') - zone = params.get('zone') - # deal with possibly passed in logical volume: - if logical_volume != None: - cfg.set('EBS', 'logical_volume_name', logical_volume.name) - cfg_fp = StringIO.StringIO() - cfg.write(cfg_fp) - # deal with the possibility that zone and/or keypair are strings read from the config file: - if isinstance(zone, Zone): - zone = zone.name - if isinstance(kp, KeyPair): - kp = kp.name - reservation = ami.run(min_count=1, - max_count=params.get('quantity', 1), - key_name=kp, - security_groups=[group], - instance_type=params.get('instance_type'), - placement = zone, - user_data = cfg_fp.getvalue()) - l = [] - i = 0 - elastic_ip = params.get('elastic_ip') - instances = reservation.instances - if elastic_ip != None and instances.__len__() > 0: - instance = instances[0] - print 'Waiting for instance to start so we can set its elastic IP address...' - # Sometimes we get a message from ec2 that says that the instance does not exist. - # Hopefully the following delay will giv eec2 enough time to get to a stable state: - time.sleep(5) - while instance.update() != 'running': - time.sleep(1) - instance.use_ip(elastic_ip) - print 'set the elastic IP of the first instance to %s' % elastic_ip - for instance in instances: - s = cls() - s.ec2 = ec2 - s.name = params.get('name') + '' if i==0 else str(i) - s.description = params.get('description') - s.region_name = region.name - s.instance_id = instance.id - if elastic_ip and i == 0: - s.elastic_ip = elastic_ip - s.put() - l.append(s) - i += 1 - return l - - @classmethod - def create_from_instance_id(cls, instance_id, name, description=''): - regions = boto.ec2.regions() - for region in regions: - ec2 = region.connect() - try: - rs = ec2.get_all_instances([instance_id]) - except: - rs = [] - if len(rs) == 1: - s = cls() - s.ec2 = ec2 - s.name = name - s.description = description - s.region_name = region.name - s.instance_id = instance_id - s._reservation = rs[0] - for instance in s._reservation.instances: - if instance.id == instance_id: - s._instance = instance - s.put() - return s - return None - - @classmethod - def create_from_current_instances(cls): - servers = [] - regions = boto.ec2.regions() - for region in regions: - ec2 = region.connect() - rs = ec2.get_all_instances() - for reservation in rs: - for instance in reservation.instances: - try: - Server.find(instance_id=instance.id).next() - boto.log.info('Server for %s already exists' % instance.id) - except StopIteration: - s = cls() - s.ec2 = ec2 - s.name = instance.id - s.region_name = region.name - s.instance_id = instance.id - s._reservation = reservation - s.put() - servers.append(s) - return servers - - def __init__(self, id=None, **kw): - Model.__init__(self, id, **kw) - self.ssh_key_file = None - self.ec2 = None - self._cmdshell = None - self._reservation = None - self._instance = None - self._setup_ec2() - - def _setup_ec2(self): - if self.ec2 and self._instance and self._reservation: - return - if self.id: - if self.region_name: - for region in boto.ec2.regions(): - if region.name == self.region_name: - self.ec2 = region.connect() - if self.instance_id and not self._instance: - try: - rs = self.ec2.get_all_instances([self.instance_id]) - if len(rs) >= 1: - for instance in rs[0].instances: - if instance.id == self.instance_id: - self._reservation = rs[0] - self._instance = instance - except EC2ResponseError: - pass - - def _status(self): - status = '' - if self._instance: - self._instance.update() - status = self._instance.state - return status - - def _hostname(self): - hostname = '' - if self._instance: - hostname = self._instance.public_dns_name - return hostname - - def _private_hostname(self): - hostname = '' - if self._instance: - hostname = self._instance.private_dns_name - return hostname - - def _instance_type(self): - it = '' - if self._instance: - it = self._instance.instance_type - return it - - def _launch_time(self): - lt = '' - if self._instance: - lt = self._instance.launch_time - return lt - - def _console_output(self): - co = '' - if self._instance: - co = self._instance.get_console_output() - return co - - def _groups(self): - gn = [] - if self._reservation: - gn = self._reservation.groups - return gn - - def _security_group(self): - groups = self._groups() - if len(groups) >= 1: - return groups[0].id - return "" - - def _zone(self): - zone = None - if self._instance: - zone = self._instance.placement - return zone - - def _key_name(self): - kn = None - if self._instance: - kn = self._instance.key_name - return kn - - def put(self): - Model.put(self) - self._setup_ec2() - - def delete(self): - if self.production: - raise ValueError, "Can't delete a production server" - #self.stop() - Model.delete(self) - - def stop(self): - if self.production: - raise ValueError, "Can't delete a production server" - if self._instance: - self._instance.stop() - - def terminate(self): - if self.production: - raise ValueError, "Can't delete a production server" - if self._instance: - self._instance.terminate() - - def reboot(self): - if self._instance: - self._instance.reboot() - - def wait(self): - while self.status != 'running': - time.sleep(5) - - def get_ssh_key_file(self): - if not self.ssh_key_file: - ssh_dir = os.path.expanduser('~/.ssh') - if os.path.isdir(ssh_dir): - ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name) - if os.path.isfile(ssh_file): - self.ssh_key_file = ssh_file - if not self.ssh_key_file: - iobject = IObject() - self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file') - return self.ssh_key_file - - def get_cmdshell(self): - if not self._cmdshell: - import cmdshell - self.get_ssh_key_file() - self._cmdshell = cmdshell.start(self) - return self._cmdshell - - def reset_cmdshell(self): - self._cmdshell = None - - def run(self, command): - with closing(self.get_cmdshell()) as cmd: - status = cmd.run(command) - return status - - def get_bundler(self, uname='root'): - self.get_ssh_key_file() - return Bundler(self, uname) - - def get_ssh_client(self, uname='root', ssh_pwd=None): - from boto.manage.cmdshell import SSHClient - self.get_ssh_key_file() - return SSHClient(self, uname=uname, ssh_pwd=ssh_pwd) - - def install(self, pkg): - return self.run('apt-get -y install %s' % pkg) - - - diff --git a/backup/src/boto/manage/task.py b/backup/src/boto/manage/task.py deleted file mode 100644 index 2f9d7d0..0000000 --- a/backup/src/boto/manage/task.py +++ /dev/null @@ -1,175 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import boto -from boto.sdb.db.property import StringProperty, DateTimeProperty, IntegerProperty -from boto.sdb.db.model import Model -import datetime, subprocess, StringIO, time - -def check_hour(val): - if val == '*': - return - if int(val) < 0 or int(val) > 23: - raise ValueError - -class Task(Model): - - """ - A scheduled, repeating task that can be executed by any participating servers. - The scheduling is similar to cron jobs. Each task has an hour attribute. - The allowable values for hour are [0-23|*]. - - To keep the operation reasonably efficient and not cause excessive polling, - the minimum granularity of a Task is hourly. Some examples: - - hour='*' - the task would be executed each hour - hour='3' - the task would be executed at 3AM GMT each day. - - """ - name = StringProperty() - hour = StringProperty(required=True, validator=check_hour, default='*') - command = StringProperty(required=True) - last_executed = DateTimeProperty() - last_status = IntegerProperty() - last_output = StringProperty() - message_id = StringProperty() - - @classmethod - def start_all(cls, queue_name): - for task in cls.all(): - task.start(queue_name) - - def __init__(self, id=None, **kw): - Model.__init__(self, id, **kw) - self.hourly = self.hour == '*' - self.daily = self.hour != '*' - self.now = datetime.datetime.utcnow() - - def check(self): - """ - Determine how long until the next scheduled time for a Task. - Returns the number of seconds until the next scheduled time or zero - if the task needs to be run immediately. - If it's an hourly task and it's never been run, run it now. - If it's a daily task and it's never been run and the hour is right, run it now. - """ - boto.log.info('checking Task[%s]-now=%s, last=%s' % (self.name, self.now, self.last_executed)) - - if self.hourly and not self.last_executed: - return 0 - - if self.daily and not self.last_executed: - if int(self.hour) == self.now.hour: - return 0 - else: - return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 - - delta = self.now - self.last_executed - if self.hourly: - if delta.seconds >= 60*60: - return 0 - else: - return 60*60 - delta.seconds - else: - if int(self.hour) == self.now.hour: - if delta.days >= 1: - return 0 - else: - return 82800 # 23 hours, just to be safe - else: - return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 - - def _run(self, msg, vtimeout): - boto.log.info('Task[%s] - running:%s' % (self.name, self.command)) - log_fp = StringIO.StringIO() - process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - nsecs = 5 - current_timeout = vtimeout - while process.poll() == None: - boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout)) - if nsecs >= current_timeout: - current_timeout += vtimeout - boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout)) - if msg: - msg.change_visibility(current_timeout) - time.sleep(5) - nsecs += 5 - t = process.communicate() - log_fp.write(t[0]) - log_fp.write(t[1]) - boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue())) - self.last_executed = self.now - self.last_status = process.returncode - self.last_output = log_fp.getvalue()[0:1023] - - def run(self, msg, vtimeout=60): - delay = self.check() - boto.log.info('Task[%s] - delay=%s seconds' % (self.name, delay)) - if delay == 0: - self._run(msg, vtimeout) - queue = msg.queue - new_msg = queue.new_message(self.id) - new_msg = queue.write(new_msg) - self.message_id = new_msg.id - self.put() - boto.log.info('Task[%s] - new message id=%s' % (self.name, new_msg.id)) - msg.delete() - boto.log.info('Task[%s] - deleted message %s' % (self.name, msg.id)) - else: - boto.log.info('new_vtimeout: %d' % delay) - msg.change_visibility(delay) - - def start(self, queue_name): - boto.log.info('Task[%s] - starting with queue: %s' % (self.name, queue_name)) - queue = boto.lookup('sqs', queue_name) - msg = queue.new_message(self.id) - msg = queue.write(msg) - self.message_id = msg.id - self.put() - boto.log.info('Task[%s] - start successful' % self.name) - -class TaskPoller(object): - - def __init__(self, queue_name): - self.sqs = boto.connect_sqs() - self.queue = self.sqs.lookup(queue_name) - - def poll(self, wait=60, vtimeout=60): - while 1: - m = self.queue.read(vtimeout) - if m: - task = Task.get_by_id(m.get_body()) - if task: - if not task.message_id or m.id == task.message_id: - boto.log.info('Task[%s] - read message %s' % (task.name, m.id)) - task.run(m, vtimeout) - else: - boto.log.info('Task[%s] - found extraneous message, ignoring' % task.name) - else: - time.sleep(wait) - - - - - - diff --git a/backup/src/boto/manage/test_manage.py b/backup/src/boto/manage/test_manage.py deleted file mode 100644 index e0b032a..0000000 --- a/backup/src/boto/manage/test_manage.py +++ /dev/null @@ -1,34 +0,0 @@ -from boto.manage.server import Server -from boto.manage.volume import Volume -import time - -print '--> Creating New Volume' -volume = Volume.create() -print volume - -print '--> Creating New Server' -server_list = Server.create() -server = server_list[0] -print server - -print '----> Waiting for Server to start up' -while server.status != 'running': - print '*' - time.sleep(10) -print '----> Server is running' - -print '--> Run "df -k" on Server' -status = server.run('df -k') -print status[1] - -print '--> Now run volume.make_ready to make the volume ready to use on server' -volume.make_ready(server) - -print '--> Run "df -k" on Server' -status = server.run('df -k') -print status[1] - -print '--> Do an "ls -al" on the new filesystem' -status = server.run('ls -al %s' % volume.mount_point) -print status[1] - diff --git a/backup/src/boto/manage/volume.py b/backup/src/boto/manage/volume.py deleted file mode 100644 index 66a458f..0000000 --- a/backup/src/boto/manage/volume.py +++ /dev/null @@ -1,420 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from __future__ import with_statement -from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty -from boto.manage.server import Server -from boto.manage import propget -import boto.ec2 -import time -import traceback -from contextlib import closing -import dateutil.parser -import datetime - - -class CommandLineGetter(object): - - def get_region(self, params): - if not params.get('region', None): - prop = self.cls.find_property('region_name') - params['region'] = propget.get(prop, choices=boto.ec2.regions) - - def get_zone(self, params): - if not params.get('zone', None): - prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', - choices=self.ec2.get_all_zones) - params['zone'] = propget.get(prop) - - def get_name(self, params): - if not params.get('name', None): - prop = self.cls.find_property('name') - params['name'] = propget.get(prop) - - def get_size(self, params): - if not params.get('size', None): - prop = IntegerProperty(name='size', verbose_name='Size (GB)') - params['size'] = propget.get(prop) - - def get_mount_point(self, params): - if not params.get('mount_point', None): - prop = self.cls.find_property('mount_point') - params['mount_point'] = propget.get(prop) - - def get_device(self, params): - if not params.get('device', None): - prop = self.cls.find_property('device') - params['device'] = propget.get(prop) - - def get(self, cls, params): - self.cls = cls - self.get_region(params) - self.ec2 = params['region'].connect() - self.get_zone(params) - self.get_name(params) - self.get_size(params) - self.get_mount_point(params) - self.get_device(params) - -class Volume(Model): - - name = StringProperty(required=True, unique=True, verbose_name='Name') - region_name = StringProperty(required=True, verbose_name='EC2 Region') - zone_name = StringProperty(required=True, verbose_name='EC2 Zone') - mount_point = StringProperty(verbose_name='Mount Point') - device = StringProperty(verbose_name="Device Name", default='/dev/sdp') - volume_id = StringProperty(required=True) - past_volume_ids = ListProperty(item_type=str) - server = ReferenceProperty(Server, collection_name='volumes', - verbose_name='Server Attached To') - volume_state = CalculatedProperty(verbose_name="Volume State", - calculated_type=str, use_method=True) - attachment_state = CalculatedProperty(verbose_name="Attachment State", - calculated_type=str, use_method=True) - size = CalculatedProperty(verbose_name="Size (GB)", - calculated_type=int, use_method=True) - - @classmethod - def create(cls, **params): - getter = CommandLineGetter() - getter.get(cls, params) - region = params.get('region') - ec2 = region.connect() - zone = params.get('zone') - size = params.get('size') - ebs_volume = ec2.create_volume(size, zone.name) - v = cls() - v.ec2 = ec2 - v.volume_id = ebs_volume.id - v.name = params.get('name') - v.mount_point = params.get('mount_point') - v.device = params.get('device') - v.region_name = region.name - v.zone_name = zone.name - v.put() - return v - - @classmethod - def create_from_volume_id(cls, region_name, volume_id, name): - vol = None - ec2 = boto.ec2.connect_to_region(region_name) - rs = ec2.get_all_volumes([volume_id]) - if len(rs) == 1: - v = rs[0] - vol = cls() - vol.volume_id = v.id - vol.name = name - vol.region_name = v.region.name - vol.zone_name = v.zone - vol.put() - return vol - - def create_from_latest_snapshot(self, name, size=None): - snapshot = self.get_snapshots()[-1] - return self.create_from_snapshot(name, snapshot, size) - - def create_from_snapshot(self, name, snapshot, size=None): - if size < self.size: - size = self.size - ec2 = self.get_ec2_connection() - if self.zone_name == None or self.zone_name == '': - # deal with the migration case where the zone is not set in the logical volume: - current_volume = ec2.get_all_volumes([self.volume_id])[0] - self.zone_name = current_volume.zone - ebs_volume = ec2.create_volume(size, self.zone_name, snapshot) - v = Volume() - v.ec2 = self.ec2 - v.volume_id = ebs_volume.id - v.name = name - v.mount_point = self.mount_point - v.device = self.device - v.region_name = self.region_name - v.zone_name = self.zone_name - v.put() - return v - - def get_ec2_connection(self): - if self.server: - return self.server.ec2 - if not hasattr(self, 'ec2') or self.ec2 == None: - self.ec2 = boto.ec2.connect_to_region(self.region_name) - return self.ec2 - - def _volume_state(self): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - return rs[0].volume_state() - - def _attachment_state(self): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - return rs[0].attachment_state() - - def _size(self): - if not hasattr(self, '__size'): - ec2 = self.get_ec2_connection() - rs = ec2.get_all_volumes([self.volume_id]) - self.__size = rs[0].size - return self.__size - - def install_xfs(self): - if self.server: - self.server.install('xfsprogs xfsdump') - - def get_snapshots(self): - """ - Returns a list of all completed snapshots for this volume ID. - """ - ec2 = self.get_ec2_connection() - rs = ec2.get_all_snapshots() - all_vols = [self.volume_id] + self.past_volume_ids - snaps = [] - for snapshot in rs: - if snapshot.volume_id in all_vols: - if snapshot.progress == '100%': - snapshot.date = dateutil.parser.parse(snapshot.start_time) - snapshot.keep = True - snaps.append(snapshot) - snaps.sort(cmp=lambda x,y: cmp(x.date, y.date)) - return snaps - - def attach(self, server=None): - if self.attachment_state == 'attached': - print 'already attached' - return None - if server: - self.server = server - self.put() - ec2 = self.get_ec2_connection() - ec2.attach_volume(self.volume_id, self.server.instance_id, self.device) - - def detach(self, force=False): - state = self.attachment_state - if state == 'available' or state == None or state == 'detaching': - print 'already detached' - return None - ec2 = self.get_ec2_connection() - ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force) - self.server = None - self.put() - - def checkfs(self, use_cmd=None): - if self.server == None: - raise ValueError, 'server attribute must be set to run this command' - # detemine state of file system on volume, only works if attached - if use_cmd: - cmd = use_cmd - else: - cmd = self.server.get_cmdshell() - status = cmd.run('xfs_check %s' % self.device) - if not use_cmd: - cmd.close() - if status[1].startswith('bad superblock magic number 0'): - return False - return True - - def wait(self): - if self.server == None: - raise ValueError, 'server attribute must be set to run this command' - with closing(self.server.get_cmdshell()) as cmd: - # wait for the volume device to appear - cmd = self.server.get_cmdshell() - while not cmd.exists(self.device): - boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) - time.sleep(10) - - def format(self): - if self.server == None: - raise ValueError, 'server attribute must be set to run this command' - status = None - with closing(self.server.get_cmdshell()) as cmd: - if not self.checkfs(cmd): - boto.log.info('make_fs...') - status = cmd.run('mkfs -t xfs %s' % self.device) - return status - - def mount(self): - if self.server == None: - raise ValueError, 'server attribute must be set to run this command' - boto.log.info('handle_mount_point') - with closing(self.server.get_cmdshell()) as cmd: - cmd = self.server.get_cmdshell() - if not cmd.isdir(self.mount_point): - boto.log.info('making directory') - # mount directory doesn't exist so create it - cmd.run("mkdir %s" % self.mount_point) - else: - boto.log.info('directory exists already') - status = cmd.run('mount -l') - lines = status[1].split('\n') - for line in lines: - t = line.split() - if t and t[2] == self.mount_point: - # something is already mounted at the mount point - # unmount that and mount it as /tmp - if t[0] != self.device: - cmd.run('umount %s' % self.mount_point) - cmd.run('mount %s /tmp' % t[0]) - cmd.run('chmod 777 /tmp') - break - # Mount up our new EBS volume onto mount_point - cmd.run("mount %s %s" % (self.device, self.mount_point)) - cmd.run('xfs_growfs %s' % self.mount_point) - - def make_ready(self, server): - self.server = server - self.put() - self.install_xfs() - self.attach() - self.wait() - self.format() - self.mount() - - def freeze(self): - if self.server: - return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point) - - def unfreeze(self): - if self.server: - return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point) - - def snapshot(self): - # if this volume is attached to a server - # we need to freeze the XFS file system - try: - self.freeze() - if self.server == None: - snapshot = self.get_ec2_connection().create_snapshot(self.volume_id) - else: - snapshot = self.server.ec2.create_snapshot(self.volume_id) - boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) - except Exception: - boto.log.info('Snapshot error') - boto.log.info(traceback.format_exc()) - finally: - status = self.unfreeze() - return status - - def get_snapshot_range(self, snaps, start_date=None, end_date=None): - l = [] - for snap in snaps: - if start_date and end_date: - if snap.date >= start_date and snap.date <= end_date: - l.append(snap) - elif start_date: - if snap.date >= start_date: - l.append(snap) - elif end_date: - if snap.date <= end_date: - l.append(snap) - else: - l.append(snap) - return l - - def trim_snapshots(self, delete=False): - """ - Trim the number of snapshots for this volume. This method always - keeps the oldest snapshot. It then uses the parameters passed in - to determine how many others should be kept. - - The algorithm is to keep all snapshots from the current day. Then - it will keep the first snapshot of the day for the previous seven days. - Then, it will keep the first snapshot of the week for the previous - four weeks. After than, it will keep the first snapshot of the month - for as many months as there are. - - """ - snaps = self.get_snapshots() - # Always keep the oldest and the newest - if len(snaps) <= 2: - return snaps - snaps = snaps[1:-1] - now = datetime.datetime.now(snaps[0].date.tzinfo) - midnight = datetime.datetime(year=now.year, month=now.month, - day=now.day, tzinfo=now.tzinfo) - # Keep the first snapshot from each day of the previous week - one_week = datetime.timedelta(days=7, seconds=60*60) - print midnight-one_week, midnight - previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight) - print previous_week - if not previous_week: - return snaps - current_day = None - for snap in previous_week: - if current_day and current_day == snap.date.day: - snap.keep = False - else: - current_day = snap.date.day - # Get ourselves onto the next full week boundary - if previous_week: - week_boundary = previous_week[0].date - if week_boundary.weekday() != 0: - delta = datetime.timedelta(days=week_boundary.weekday()) - week_boundary = week_boundary - delta - # Keep one within this partial week - partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date) - if len(partial_week) > 1: - for snap in partial_week[1:]: - snap.keep = False - # Keep the first snapshot of each week for the previous 4 weeks - for i in range(0,4): - weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary) - if len(weeks_worth) > 1: - for snap in weeks_worth[1:]: - snap.keep = False - week_boundary = week_boundary - one_week - # Now look through all remaining snaps and keep one per month - remainder = self.get_snapshot_range(snaps, end_date=week_boundary) - current_month = None - for snap in remainder: - if current_month and current_month == snap.date.month: - snap.keep = False - else: - current_month = snap.date.month - if delete: - for snap in snaps: - if not snap.keep: - boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name)) - snap.delete() - return snaps - - def grow(self, size): - pass - - def copy(self, snapshot): - pass - - def get_snapshot_from_date(self, date): - pass - - def delete(self, delete_ebs_volume=False): - if delete_ebs_volume: - self.detach() - ec2 = self.get_ec2_connection() - ec2.delete_volume(self.volume_id) - Model.delete(self) - - def archive(self): - # snapshot volume, trim snaps, delete volume-id - pass - - diff --git a/backup/src/boto/mashups/__init__.py b/backup/src/boto/mashups/__init__.py deleted file mode 100644 index 449bd16..0000000 --- a/backup/src/boto/mashups/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - - diff --git a/backup/src/boto/mashups/interactive.py b/backup/src/boto/mashups/interactive.py deleted file mode 100644 index b80e661..0000000 --- a/backup/src/boto/mashups/interactive.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (C) 2003-2007 Robey Pointer -# -# This file is part of paramiko. -# -# Paramiko is free software; you can redistribute it and/or modify it under the -# terms of the GNU Lesser General Public License as published by the Free -# Software Foundation; either version 2.1 of the License, or (at your option) -# any later version. -# -# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more -# details. -# -# You should have received a copy of the GNU Lesser General Public License -# along with Paramiko; if not, write to the Free Software Foundation, Inc., -# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. - - -import socket -import sys - -# windows does not have termios... -try: - import termios - import tty - has_termios = True -except ImportError: - has_termios = False - - -def interactive_shell(chan): - if has_termios: - posix_shell(chan) - else: - windows_shell(chan) - - -def posix_shell(chan): - import select - - oldtty = termios.tcgetattr(sys.stdin) - try: - tty.setraw(sys.stdin.fileno()) - tty.setcbreak(sys.stdin.fileno()) - chan.settimeout(0.0) - - while True: - r, w, e = select.select([chan, sys.stdin], [], []) - if chan in r: - try: - x = chan.recv(1024) - if len(x) == 0: - print '\r\n*** EOF\r\n', - break - sys.stdout.write(x) - sys.stdout.flush() - except socket.timeout: - pass - if sys.stdin in r: - x = sys.stdin.read(1) - if len(x) == 0: - break - chan.send(x) - - finally: - termios.tcsetattr(sys.stdin, termios.TCSADRAIN, oldtty) - - -# thanks to Mike Looijmans for this code -def windows_shell(chan): - import threading - - sys.stdout.write("Line-buffered terminal emulation. Press F6 or ^Z to send EOF.\r\n\r\n") - - def writeall(sock): - while True: - data = sock.recv(256) - if not data: - sys.stdout.write('\r\n*** EOF ***\r\n\r\n') - sys.stdout.flush() - break - sys.stdout.write(data) - sys.stdout.flush() - - writer = threading.Thread(target=writeall, args=(chan,)) - writer.start() - - try: - while True: - d = sys.stdin.read(1) - if not d: - break - chan.send(d) - except EOFError: - # user hit ^Z or F6 - pass diff --git a/backup/src/boto/mashups/iobject.py b/backup/src/boto/mashups/iobject.py deleted file mode 100644 index a226b5c..0000000 --- a/backup/src/boto/mashups/iobject.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import os - -def int_val_fn(v): - try: - int(v) - return True - except: - return False - -class IObject(object): - - def choose_from_list(self, item_list, search_str='', - prompt='Enter Selection'): - if not item_list: - print 'No Choices Available' - return - choice = None - while not choice: - n = 1 - choices = [] - for item in item_list: - if isinstance(item, str): - print '[%d] %s' % (n, item) - choices.append(item) - n += 1 - else: - obj, id, desc = item - if desc: - if desc.find(search_str) >= 0: - print '[%d] %s - %s' % (n, id, desc) - choices.append(obj) - n += 1 - else: - if id.find(search_str) >= 0: - print '[%d] %s' % (n, id) - choices.append(obj) - n += 1 - if choices: - val = raw_input('%s[1-%d]: ' % (prompt, len(choices))) - if val.startswith('/'): - search_str = val[1:] - else: - try: - int_val = int(val) - if int_val == 0: - return None - choice = choices[int_val-1] - except ValueError: - print '%s is not a valid choice' % val - except IndexError: - print '%s is not within the range[1-%d]' % (val, - len(choices)) - else: - print "No objects matched your pattern" - search_str = '' - return choice - - def get_string(self, prompt, validation_fn=None): - okay = False - while not okay: - val = raw_input('%s: ' % prompt) - if validation_fn: - okay = validation_fn(val) - if not okay: - print 'Invalid value: %s' % val - else: - okay = True - return val - - def get_filename(self, prompt): - okay = False - val = '' - while not okay: - val = raw_input('%s: %s' % (prompt, val)) - val = os.path.expanduser(val) - if os.path.isfile(val): - okay = True - elif os.path.isdir(val): - path = val - val = self.choose_from_list(os.listdir(path)) - if val: - val = os.path.join(path, val) - okay = True - else: - val = '' - else: - print 'Invalid value: %s' % val - val = '' - return val - - def get_int(self, prompt): - s = self.get_string(prompt, int_val_fn) - return int(s) - diff --git a/backup/src/boto/mashups/order.py b/backup/src/boto/mashups/order.py deleted file mode 100644 index 6efdc3e..0000000 --- a/backup/src/boto/mashups/order.py +++ /dev/null @@ -1,211 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -High-level abstraction of an EC2 order for servers -""" - -import boto -import boto.ec2 -from boto.mashups.server import Server, ServerSet -from boto.mashups.iobject import IObject -from boto.pyami.config import Config -from boto.sdb.persist import get_domain, set_domain -import time, StringIO - -InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] - -class Item(IObject): - - def __init__(self): - self.region = None - self.name = None - self.instance_type = None - self.quantity = 0 - self.zone = None - self.ami = None - self.groups = [] - self.key = None - self.ec2 = None - self.config = None - - def set_userdata(self, key, value): - self.userdata[key] = value - - def get_userdata(self, key): - return self.userdata[key] - - def set_region(self, region=None): - if region: - self.region = region - else: - l = [(r, r.name, r.endpoint) for r in boto.ec2.regions()] - self.region = self.choose_from_list(l, prompt='Choose Region') - - def set_name(self, name=None): - if name: - self.name = name - else: - self.name = self.get_string('Name') - - def set_instance_type(self, instance_type=None): - if instance_type: - self.instance_type = instance_type - else: - self.instance_type = self.choose_from_list(InstanceTypes, 'Instance Type') - - def set_quantity(self, n=0): - if n > 0: - self.quantity = n - else: - self.quantity = self.get_int('Quantity') - - def set_zone(self, zone=None): - if zone: - self.zone = zone - else: - l = [(z, z.name, z.state) for z in self.ec2.get_all_zones()] - self.zone = self.choose_from_list(l, prompt='Choose Availability Zone') - - def set_ami(self, ami=None): - if ami: - self.ami = ami - else: - l = [(a, a.id, a.location) for a in self.ec2.get_all_images()] - self.ami = self.choose_from_list(l, prompt='Choose AMI') - - def add_group(self, group=None): - if group: - self.groups.append(group) - else: - l = [(s, s.name, s.description) for s in self.ec2.get_all_security_groups()] - self.groups.append(self.choose_from_list(l, prompt='Choose Security Group')) - - def set_key(self, key=None): - if key: - self.key = key - else: - l = [(k, k.name, '') for k in self.ec2.get_all_key_pairs()] - self.key = self.choose_from_list(l, prompt='Choose Keypair') - - def update_config(self): - if not self.config.has_section('Credentials'): - self.config.add_section('Credentials') - self.config.set('Credentials', 'aws_access_key_id', self.ec2.aws_access_key_id) - self.config.set('Credentials', 'aws_secret_access_key', self.ec2.aws_secret_access_key) - if not self.config.has_section('Pyami'): - self.config.add_section('Pyami') - sdb_domain = get_domain() - if sdb_domain: - self.config.set('Pyami', 'server_sdb_domain', sdb_domain) - self.config.set('Pyami', 'server_sdb_name', self.name) - - def set_config(self, config_path=None): - if not config_path: - config_path = self.get_filename('Specify Config file') - self.config = Config(path=config_path) - - def get_userdata_string(self): - s = StringIO.StringIO() - self.config.write(s) - return s.getvalue() - - def enter(self, **params): - self.region = params.get('region', self.region) - if not self.region: - self.set_region() - self.ec2 = self.region.connect() - self.name = params.get('name', self.name) - if not self.name: - self.set_name() - self.instance_type = params.get('instance_type', self.instance_type) - if not self.instance_type: - self.set_instance_type() - self.zone = params.get('zone', self.zone) - if not self.zone: - self.set_zone() - self.quantity = params.get('quantity', self.quantity) - if not self.quantity: - self.set_quantity() - self.ami = params.get('ami', self.ami) - if not self.ami: - self.set_ami() - self.groups = params.get('groups', self.groups) - if not self.groups: - self.add_group() - self.key = params.get('key', self.key) - if not self.key: - self.set_key() - self.config = params.get('config', self.config) - if not self.config: - self.set_config() - self.update_config() - -class Order(IObject): - - def __init__(self): - self.items = [] - self.reservation = None - - def add_item(self, **params): - item = Item() - item.enter(**params) - self.items.append(item) - - def display(self): - print 'This Order consists of the following items' - print - print 'QTY\tNAME\tTYPE\nAMI\t\tGroups\t\t\tKeyPair' - for item in self.items: - print '%s\t%s\t%s\t%s\t%s\t%s' % (item.quantity, item.name, item.instance_type, - item.ami.id, item.groups, item.key.name) - - def place(self, block=True): - if get_domain() == None: - print 'SDB Persistence Domain not set' - domain_name = self.get_string('Specify SDB Domain') - set_domain(domain_name) - s = ServerSet() - for item in self.items: - r = item.ami.run(min_count=1, max_count=item.quantity, - key_name=item.key.name, user_data=item.get_userdata_string(), - security_groups=item.groups, instance_type=item.instance_type, - placement=item.zone.name) - if block: - states = [i.state for i in r.instances] - if states.count('running') != len(states): - print states - time.sleep(15) - states = [i.update() for i in r.instances] - for i in r.instances: - server = Server() - server.name = item.name - server.instance_id = i.id - server.reservation = r - server.save() - s.append(server) - if len(s) == 1: - return s[0] - else: - return s - - - diff --git a/backup/src/boto/mashups/server.py b/backup/src/boto/mashups/server.py deleted file mode 100644 index 6cea106..0000000 --- a/backup/src/boto/mashups/server.py +++ /dev/null @@ -1,395 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -High-level abstraction of an EC2 server -""" -import boto -import boto.utils -from boto.mashups.iobject import IObject -from boto.pyami.config import Config, BotoConfigPath -from boto.mashups.interactive import interactive_shell -from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty -import os -import StringIO - - -class ServerSet(list): - - def __getattr__(self, name): - results = [] - is_callable = False - for server in self: - try: - val = getattr(server, name) - if callable(val): - is_callable = True - results.append(val) - except: - results.append(None) - if is_callable: - self.map_list = results - return self.map - return results - - def map(self, *args): - results = [] - for fn in self.map_list: - results.append(fn(*args)) - return results - -class Server(Model): - - @property - def ec2(self): - if self._ec2 is None: - self._ec2 = boto.connect_ec2() - return self._ec2 - - @classmethod - def Inventory(cls): - """ - Returns a list of Server instances, one for each Server object - persisted in the db - """ - l = ServerSet() - rs = cls.find() - for server in rs: - l.append(server) - return l - - @classmethod - def Register(cls, name, instance_id, description=''): - s = cls() - s.name = name - s.instance_id = instance_id - s.description = description - s.save() - return s - - def __init__(self, id=None, **kw): - Model.__init__(self, id, **kw) - self._reservation = None - self._instance = None - self._ssh_client = None - self._pkey = None - self._config = None - self._ec2 = None - - name = StringProperty(unique=True, verbose_name="Name") - instance_id = StringProperty(verbose_name="Instance ID") - config_uri = StringProperty() - ami_id = StringProperty(verbose_name="AMI ID") - zone = StringProperty(verbose_name="Availability Zone") - security_group = StringProperty(verbose_name="Security Group", default="default") - key_name = StringProperty(verbose_name="Key Name") - elastic_ip = StringProperty(verbose_name="Elastic IP") - instance_type = StringProperty(verbose_name="Instance Type") - description = StringProperty(verbose_name="Description") - log = StringProperty() - - def setReadOnly(self, value): - raise AttributeError - - def getInstance(self): - if not self._instance: - if self.instance_id: - try: - rs = self.ec2.get_all_instances([self.instance_id]) - except: - return None - if len(rs) > 0: - self._reservation = rs[0] - self._instance = self._reservation.instances[0] - return self._instance - - instance = property(getInstance, setReadOnly, None, 'The Instance for the server') - - def getAMI(self): - if self.instance: - return self.instance.image_id - - ami = property(getAMI, setReadOnly, None, 'The AMI for the server') - - def getStatus(self): - if self.instance: - self.instance.update() - return self.instance.state - - status = property(getStatus, setReadOnly, None, - 'The status of the server') - - def getHostname(self): - if self.instance: - return self.instance.public_dns_name - - hostname = property(getHostname, setReadOnly, None, - 'The public DNS name of the server') - - def getPrivateHostname(self): - if self.instance: - return self.instance.private_dns_name - - private_hostname = property(getPrivateHostname, setReadOnly, None, - 'The private DNS name of the server') - - def getLaunchTime(self): - if self.instance: - return self.instance.launch_time - - launch_time = property(getLaunchTime, setReadOnly, None, - 'The time the Server was started') - - def getConsoleOutput(self): - if self.instance: - return self.instance.get_console_output() - - console_output = property(getConsoleOutput, setReadOnly, None, - 'Retrieve the console output for server') - - def getGroups(self): - if self._reservation: - return self._reservation.groups - else: - return None - - groups = property(getGroups, setReadOnly, None, - 'The Security Groups controlling access to this server') - - def getConfig(self): - if not self._config: - remote_file = BotoConfigPath - local_file = '%s.ini' % self.instance.id - self.get_file(remote_file, local_file) - self._config = Config(local_file) - return self._config - - def setConfig(self, config): - local_file = '%s.ini' % self.instance.id - fp = open(local_file) - config.write(fp) - fp.close() - self.put_file(local_file, BotoConfigPath) - self._config = config - - config = property(getConfig, setConfig, None, - 'The instance data for this server') - - def set_config(self, config): - """ - Set SDB based config - """ - self._config = config - self._config.dump_to_sdb("botoConfigs", self.id) - - def load_config(self): - self._config = Config(do_load=False) - self._config.load_from_sdb("botoConfigs", self.id) - - def stop(self): - if self.instance: - self.instance.stop() - - def start(self): - self.stop() - ec2 = boto.connect_ec2() - ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0] - groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)]) - if not self._config: - self.load_config() - if not self._config.has_section("Credentials"): - self._config.add_section("Credentials") - self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id) - self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key) - - if not self._config.has_section("Pyami"): - self._config.add_section("Pyami") - - if self._manager.domain: - self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name) - self._config.set("Pyami", 'server_sdb_name', self.name) - - cfg = StringIO.StringIO() - self._config.write(cfg) - cfg = cfg.getvalue() - r = ami.run(min_count=1, - max_count=1, - key_name=self.key_name, - security_groups = groups, - instance_type = self.instance_type, - placement = self.zone, - user_data = cfg) - i = r.instances[0] - self.instance_id = i.id - self.put() - if self.elastic_ip: - ec2.associate_address(self.instance_id, self.elastic_ip) - - def reboot(self): - if self.instance: - self.instance.reboot() - - def get_ssh_client(self, key_file=None, host_key_file='~/.ssh/known_hosts', - uname='root'): - import paramiko - if not self.instance: - print 'No instance yet!' - return - if not self._ssh_client: - if not key_file: - iobject = IObject() - key_file = iobject.get_filename('Path to OpenSSH Key file') - self._pkey = paramiko.RSAKey.from_private_key_file(key_file) - self._ssh_client = paramiko.SSHClient() - self._ssh_client.load_system_host_keys() - self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) - self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - self._ssh_client.connect(self.instance.public_dns_name, - username=uname, pkey=self._pkey) - return self._ssh_client - - def get_file(self, remotepath, localpath): - ssh_client = self.get_ssh_client() - sftp_client = ssh_client.open_sftp() - sftp_client.get(remotepath, localpath) - - def put_file(self, localpath, remotepath): - ssh_client = self.get_ssh_client() - sftp_client = ssh_client.open_sftp() - sftp_client.put(localpath, remotepath) - - def listdir(self, remotepath): - ssh_client = self.get_ssh_client() - sftp_client = ssh_client.open_sftp() - return sftp_client.listdir(remotepath) - - def shell(self, key_file=None): - ssh_client = self.get_ssh_client(key_file) - channel = ssh_client.invoke_shell() - interactive_shell(channel) - - def bundle_image(self, prefix, key_file, cert_file, size): - print 'bundling image...' - print '\tcopying cert and pk over to /mnt directory on server' - ssh_client = self.get_ssh_client() - sftp_client = ssh_client.open_sftp() - path, name = os.path.split(key_file) - remote_key_file = '/mnt/%s' % name - self.put_file(key_file, remote_key_file) - path, name = os.path.split(cert_file) - remote_cert_file = '/mnt/%s' % name - self.put_file(cert_file, remote_cert_file) - print '\tdeleting %s' % BotoConfigPath - # delete the metadata.ini file if it exists - try: - sftp_client.remove(BotoConfigPath) - except: - pass - command = 'sudo ec2-bundle-vol ' - command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) - command += '-u %s ' % self._reservation.owner_id - command += '-p %s ' % prefix - command += '-s %d ' % size - command += '-d /mnt ' - if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium': - command += '-r i386' - else: - command += '-r x86_64' - print '\t%s' % command - t = ssh_client.exec_command(command) - response = t[1].read() - print '\t%s' % response - print '\t%s' % t[2].read() - print '...complete!' - - def upload_bundle(self, bucket, prefix): - print 'uploading bundle...' - command = 'ec2-upload-bundle ' - command += '-m /mnt/%s.manifest.xml ' % prefix - command += '-b %s ' % bucket - command += '-a %s ' % self.ec2.aws_access_key_id - command += '-s %s ' % self.ec2.aws_secret_access_key - print '\t%s' % command - ssh_client = self.get_ssh_client() - t = ssh_client.exec_command(command) - response = t[1].read() - print '\t%s' % response - print '\t%s' % t[2].read() - print '...complete!' - - def create_image(self, bucket=None, prefix=None, key_file=None, cert_file=None, size=None): - iobject = IObject() - if not bucket: - bucket = iobject.get_string('Name of S3 bucket') - if not prefix: - prefix = iobject.get_string('Prefix for AMI file') - if not key_file: - key_file = iobject.get_filename('Path to RSA private key file') - if not cert_file: - cert_file = iobject.get_filename('Path to RSA public cert file') - if not size: - size = iobject.get_int('Size (in MB) of bundled image') - self.bundle_image(prefix, key_file, cert_file, size) - self.upload_bundle(bucket, prefix) - print 'registering image...' - self.image_id = self.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) - return self.image_id - - def attach_volume(self, volume, device="/dev/sdp"): - """ - Attach an EBS volume to this server - - :param volume: EBS Volume to attach - :type volume: boto.ec2.volume.Volume - - :param device: Device to attach to (default to /dev/sdp) - :type device: string - """ - if hasattr(volume, "id"): - volume_id = volume.id - else: - volume_id = volume - return self.ec2.attach_volume(volume_id=volume_id, instance_id=self.instance_id, device=device) - - def detach_volume(self, volume): - """ - Detach an EBS volume from this server - - :param volume: EBS Volume to detach - :type volume: boto.ec2.volume.Volume - """ - if hasattr(volume, "id"): - volume_id = volume.id - else: - volume_id = volume - return self.ec2.detach_volume(volume_id=volume_id, instance_id=self.instance_id) - - def install_package(self, package_name): - print 'installing %s...' % package_name - command = 'yum -y install %s' % package_name - print '\t%s' % command - ssh_client = self.get_ssh_client() - t = ssh_client.exec_command(command) - response = t[1].read() - print '\t%s' % response - print '\t%s' % t[2].read() - print '...complete!' diff --git a/backup/src/boto/mturk/__init__.py b/backup/src/boto/mturk/__init__.py deleted file mode 100644 index 449bd16..0000000 --- a/backup/src/boto/mturk/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - - diff --git a/backup/src/boto/mturk/connection.py b/backup/src/boto/mturk/connection.py deleted file mode 100644 index 619697f..0000000 --- a/backup/src/boto/mturk/connection.py +++ /dev/null @@ -1,887 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import xml.sax -import datetime -import itertools - -from boto import handler -from boto import config -from boto.mturk.price import Price -import boto.mturk.notification -from boto.connection import AWSQueryConnection -from boto.exception import EC2ResponseError -from boto.resultset import ResultSet -from boto.mturk.question import QuestionForm, ExternalQuestion - -class MTurkRequestError(EC2ResponseError): - "Error for MTurk Requests" - # todo: subclass from an abstract parent of EC2ResponseError - -class MTurkConnection(AWSQueryConnection): - - APIVersion = '2008-08-02' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=False, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, - host=None, debug=0, - https_connection_factory=None): - if not host: - if config.has_option('MTurk', 'sandbox') and config.get('MTurk', 'sandbox') == 'True': - host = 'mechanicalturk.sandbox.amazonaws.com' - else: - host = 'mechanicalturk.amazonaws.com' - - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, proxy_port, - proxy_user, proxy_pass, host, debug, - https_connection_factory) - - def _required_auth_capability(self): - return ['mturk'] - - def get_account_balance(self): - """ - """ - params = {} - return self._process_request('GetAccountBalance', params, - [('AvailableBalance', Price), - ('OnHoldBalance', Price)]) - - def register_hit_type(self, title, description, reward, duration, - keywords=None, approval_delay=None, qual_req=None): - """ - Register a new HIT Type - title, description are strings - reward is a Price object - duration can be a timedelta, or an object castable to an int - """ - params = dict( - Title=title, - Description=description, - AssignmentDurationInSeconds= - self.duration_as_seconds(duration), - ) - params.update(MTurkConnection.get_price_as_price(reward).get_as_params('Reward')) - - if keywords: - params['Keywords'] = self.get_keywords_as_string(keywords) - - if approval_delay is not None: - d = self.duration_as_seconds(approval_delay) - params['AutoApprovalDelayInSeconds'] = d - - if qual_req is not None: - params.update(qual_req.get_as_params()) - - return self._process_request('RegisterHITType', params) - - def set_email_notification(self, hit_type, email, event_types=None): - """ - Performs a SetHITTypeNotification operation to set email - notification for a specified HIT type - """ - return self._set_notification(hit_type, 'Email', email, event_types) - - def set_rest_notification(self, hit_type, url, event_types=None): - """ - Performs a SetHITTypeNotification operation to set REST notification - for a specified HIT type - """ - return self._set_notification(hit_type, 'REST', url, event_types) - - def _set_notification(self, hit_type, transport, destination, event_types=None): - """ - Common SetHITTypeNotification operation to set notification for a - specified HIT type - """ - assert type(hit_type) is str, "hit_type argument should be a string." - - params = {'HITTypeId': hit_type} - - # from the Developer Guide: - # The 'Active' parameter is optional. If omitted, the active status of - # the HIT type's notification specification is unchanged. All HIT types - # begin with their notification specifications in the "inactive" status. - notification_params = {'Destination': destination, - 'Transport': transport, - 'Version': boto.mturk.notification.NotificationMessage.NOTIFICATION_VERSION, - 'Active': True, - } - - # add specific event types if required - if event_types: - self.build_list_params(notification_params, event_types, 'EventType') - - # Set up dict of 'Notification.1.Transport' etc. values - notification_rest_params = {} - num = 1 - for key in notification_params: - notification_rest_params['Notification.%d.%s' % (num, key)] = notification_params[key] - - # Update main params dict - params.update(notification_rest_params) - - # Execute operation - return self._process_request('SetHITTypeNotification', params) - - def create_hit(self, hit_type=None, question=None, - lifetime=datetime.timedelta(days=7), - max_assignments=1, - title=None, description=None, keywords=None, - reward=None, duration=datetime.timedelta(days=7), - approval_delay=None, annotation=None, - questions=None, qualifications=None, - response_groups=None): - """ - Creates a new HIT. - Returns a ResultSet - See: http://docs.amazonwebservices.com/AWSMechanicalTurkRequester/2006-10-31/ApiReference_CreateHITOperation.html - """ - - # handle single or multiple questions - neither = question is None and questions is None - both = question is not None and questions is not None - if neither or both: - raise ValueError("Must specify either question (single Question instance) or questions (list or QuestionForm instance), but not both") - - if question: - questions = [question] - question_param = QuestionForm(questions) - if isinstance(question, QuestionForm): - question_param = question - elif isinstance(question, ExternalQuestion): - question_param = question - - # Handle basic required arguments and set up params dict - params = {'Question': question_param.get_as_xml(), - 'LifetimeInSeconds' : - self.duration_as_seconds(lifetime), - 'MaxAssignments' : max_assignments, - } - - # if hit type specified then add it - # else add the additional required parameters - if hit_type: - params['HITTypeId'] = hit_type - else: - # Handle keywords - final_keywords = MTurkConnection.get_keywords_as_string(keywords) - - # Handle price argument - final_price = MTurkConnection.get_price_as_price(reward) - - final_duration = self.duration_as_seconds(duration) - - additional_params = dict( - Title=title, - Description=description, - Keywords=final_keywords, - AssignmentDurationInSeconds=final_duration, - ) - additional_params.update(final_price.get_as_params('Reward')) - - if approval_delay is not None: - d = self.duration_as_seconds(approval_delay) - additional_params['AutoApprovalDelayInSeconds'] = d - - # add these params to the others - params.update(additional_params) - - # add the annotation if specified - if annotation is not None: - params['RequesterAnnotation'] = annotation - - # Add the Qualifications if specified - if qualifications is not None: - params.update(qualifications.get_as_params()) - - # Handle optional response groups argument - if response_groups: - self.build_list_params(params, response_groups, 'ResponseGroup') - - # Submit - return self._process_request('CreateHIT', params, [('HIT', HIT),]) - - def change_hit_type_of_hit(self, hit_id, hit_type): - """ - Change the HIT type of an existing HIT. Note that the reward associated - with the new HIT type must match the reward of the current HIT type in - order for the operation to be valid. - \thit_id is a string - \thit_type is a string - """ - params = {'HITId' : hit_id, - 'HITTypeId': hit_type} - - return self._process_request('ChangeHITTypeOfHIT', params) - - def get_reviewable_hits(self, hit_type=None, status='Reviewable', - sort_by='Expiration', sort_direction='Ascending', - page_size=10, page_number=1): - """ - Retrieve the HITs that have a status of Reviewable, or HITs that - have a status of Reviewing, and that belong to the Requester - calling the operation. - """ - params = {'Status' : status, - 'SortProperty' : sort_by, - 'SortDirection' : sort_direction, - 'PageSize' : page_size, - 'PageNumber' : page_number} - - # Handle optional hit_type argument - if hit_type is not None: - params.update({'HITTypeId': hit_type}) - - return self._process_request('GetReviewableHITs', params, [('HIT', HIT),]) - - @staticmethod - def _get_pages(page_size, total_records): - """ - Given a page size (records per page) and a total number of - records, return the page numbers to be retrieved. - """ - pages = total_records/page_size+bool(total_records%page_size) - return range(1, pages+1) - - - def get_all_hits(self): - """ - Return all of a Requester's HITs - - Despite what search_hits says, it does not return all hits, but - instead returns a page of hits. This method will pull the hits - from the server 100 at a time, but will yield the results - iteratively, so subsequent requests are made on demand. - """ - page_size = 100 - search_rs = self.search_hits(page_size=page_size) - total_records = int(search_rs.TotalNumResults) - get_page_hits = lambda(page): self.search_hits(page_size=page_size, page_number=page) - page_nums = self._get_pages(page_size, total_records) - hit_sets = itertools.imap(get_page_hits, page_nums) - return itertools.chain.from_iterable(hit_sets) - - def search_hits(self, sort_by='CreationTime', sort_direction='Ascending', - page_size=10, page_number=1, response_groups=None): - """ - Return a page of a Requester's HITs, on behalf of the Requester. - The operation returns HITs of any status, except for HITs that - have been disposed with the DisposeHIT operation. - Note: - The SearchHITs operation does not accept any search parameters - that filter the results. - """ - params = {'SortProperty' : sort_by, - 'SortDirection' : sort_direction, - 'PageSize' : page_size, - 'PageNumber' : page_number} - # Handle optional response groups argument - if response_groups: - self.build_list_params(params, response_groups, 'ResponseGroup') - - - return self._process_request('SearchHITs', params, [('HIT', HIT),]) - - def get_assignments(self, hit_id, status=None, - sort_by='SubmitTime', sort_direction='Ascending', - page_size=10, page_number=1, response_groups=None): - """ - Retrieves completed assignments for a HIT. - Use this operation to retrieve the results for a HIT. - - The returned ResultSet will have the following attributes: - - NumResults - The number of assignments on the page in the filtered results - list, equivalent to the number of assignments being returned - by this call. - A non-negative integer - PageNumber - The number of the page in the filtered results list being - returned. - A positive integer - TotalNumResults - The total number of HITs in the filtered results list based - on this call. - A non-negative integer - - The ResultSet will contain zero or more Assignment objects - - """ - params = {'HITId' : hit_id, - 'SortProperty' : sort_by, - 'SortDirection' : sort_direction, - 'PageSize' : page_size, - 'PageNumber' : page_number} - - if status is not None: - params['AssignmentStatus'] = status - - # Handle optional response groups argument - if response_groups: - self.build_list_params(params, response_groups, 'ResponseGroup') - - return self._process_request('GetAssignmentsForHIT', params, - [('Assignment', Assignment),]) - - def approve_assignment(self, assignment_id, feedback=None): - """ - """ - params = {'AssignmentId' : assignment_id,} - if feedback: - params['RequesterFeedback'] = feedback - return self._process_request('ApproveAssignment', params) - - def reject_assignment(self, assignment_id, feedback=None): - """ - """ - params = {'AssignmentId' : assignment_id,} - if feedback: - params['RequesterFeedback'] = feedback - return self._process_request('RejectAssignment', params) - - def get_hit(self, hit_id, response_groups=None): - """ - """ - params = {'HITId' : hit_id,} - # Handle optional response groups argument - if response_groups: - self.build_list_params(params, response_groups, 'ResponseGroup') - - return self._process_request('GetHIT', params, [('HIT', HIT),]) - - def set_reviewing(self, hit_id, revert=None): - """ - Update a HIT with a status of Reviewable to have a status of Reviewing, - or reverts a Reviewing HIT back to the Reviewable status. - - Only HITs with a status of Reviewable can be updated with a status of - Reviewing. Similarly, only Reviewing HITs can be reverted back to a - status of Reviewable. - """ - params = {'HITId' : hit_id,} - if revert: - params['Revert'] = revert - return self._process_request('SetHITAsReviewing', params) - - def disable_hit(self, hit_id, response_groups=None): - """ - Remove a HIT from the Mechanical Turk marketplace, approves all - submitted assignments that have not already been approved or rejected, - and disposes of the HIT and all assignment data. - - Assignments for the HIT that have already been submitted, but not yet - approved or rejected, will be automatically approved. Assignments in - progress at the time of the call to DisableHIT will be approved once - the assignments are submitted. You will be charged for approval of - these assignments. DisableHIT completely disposes of the HIT and - all submitted assignment data. Assignment results data cannot be - retrieved for a HIT that has been disposed. - - It is not possible to re-enable a HIT once it has been disabled. - To make the work from a disabled HIT available again, create a new HIT. - """ - params = {'HITId' : hit_id,} - # Handle optional response groups argument - if response_groups: - self.build_list_params(params, response_groups, 'ResponseGroup') - - return self._process_request('DisableHIT', params) - - def dispose_hit(self, hit_id): - """ - Dispose of a HIT that is no longer needed. - - Only HITs in the "reviewable" state, with all submitted - assignments approved or rejected, can be disposed. A Requester - can call GetReviewableHITs to determine which HITs are - reviewable, then call GetAssignmentsForHIT to retrieve the - assignments. Disposing of a HIT removes the HIT from the - results of a call to GetReviewableHITs. """ - params = {'HITId' : hit_id,} - return self._process_request('DisposeHIT', params) - - def expire_hit(self, hit_id): - - """ - Expire a HIT that is no longer needed. - - The effect is identical to the HIT expiring on its own. The - HIT no longer appears on the Mechanical Turk web site, and no - new Workers are allowed to accept the HIT. Workers who have - accepted the HIT prior to expiration are allowed to complete - it or return it, or allow the assignment duration to elapse - (abandon the HIT). Once all remaining assignments have been - submitted, the expired HIT becomes"reviewable", and will be - returned by a call to GetReviewableHITs. - """ - params = {'HITId' : hit_id,} - return self._process_request('ForceExpireHIT', params) - - def extend_hit(self, hit_id, assignments_increment=None, expiration_increment=None): - """ - Increase the maximum number of assignments, or extend the - expiration date, of an existing HIT. - - NOTE: If a HIT has a status of Reviewable and the HIT is - extended to make it Available, the HIT will not be returned by - GetReviewableHITs, and its submitted assignments will not be - returned by GetAssignmentsForHIT, until the HIT is Reviewable - again. Assignment auto-approval will still happen on its - original schedule, even if the HIT has been extended. Be sure - to retrieve and approve (or reject) submitted assignments - before extending the HIT, if so desired. - """ - # must provide assignment *or* expiration increment - if (assignments_increment is None and expiration_increment is None) or \ - (assignments_increment is not None and expiration_increment is not None): - raise ValueError("Must specify either assignments_increment or expiration_increment, but not both") - - params = {'HITId' : hit_id,} - if assignments_increment: - params['MaxAssignmentsIncrement'] = assignments_increment - if expiration_increment: - params['ExpirationIncrementInSeconds'] = expiration_increment - - return self._process_request('ExtendHIT', params) - - def get_help(self, about, help_type='Operation'): - """ - Return information about the Mechanical Turk Service - operations and response group NOTE - this is basically useless - as it just returns the URL of the documentation - - help_type: either 'Operation' or 'ResponseGroup' - """ - params = {'About': about, 'HelpType': help_type,} - return self._process_request('Help', params) - - def grant_bonus(self, worker_id, assignment_id, bonus_price, reason): - """ - Issues a payment of money from your account to a Worker. To - be eligible for a bonus, the Worker must have submitted - results for one of your HITs, and have had those results - approved or rejected. This payment happens separately from the - reward you pay to the Worker when you approve the Worker's - assignment. The Bonus must be passed in as an instance of the - Price object. - """ - params = bonus_price.get_as_params('BonusAmount', 1) - params['WorkerId'] = worker_id - params['AssignmentId'] = assignment_id - params['Reason'] = reason - - return self._process_request('GrantBonus', params) - - def block_worker(self, worker_id, reason): - """ - Block a worker from working on my tasks. - """ - params = {'WorkerId': worker_id, 'Reason': reason} - - return self._process_request('BlockWorker', params) - - def unblock_worker(self, worker_id, reason): - """ - Unblock a worker from working on my tasks. - """ - params = {'WorkerId': worker_id, 'Reason': reason} - - return self._process_request('UnblockWorker', params) - - def notify_workers(self, worker_ids, subject, message_text): - """ - Send a text message to workers. - """ - params = {'WorkerId' : worker_ids, - 'Subject' : subject, - 'MessageText': message_text} - - return self._process_request('NotifyWorkers', params) - - def create_qualification_type(self, - name, - description, - status, - keywords=None, - retry_delay=None, - test=None, - answer_key=None, - answer_key_xml=None, - test_duration=None, - auto_granted=False, - auto_granted_value=1): - """ - Create a new Qualification Type. - - name: This will be visible to workers and must be unique for a - given requester. - - description: description shown to workers. Max 2000 characters. - - status: 'Active' or 'Inactive' - - keywords: list of keyword strings or comma separated string. - Max length of 1000 characters when concatenated with commas. - - retry_delay: number of seconds after requesting a - qualification the worker must wait before they can ask again. - If not specified, workers can only request this qualification - once. - - test: a QuestionForm - - answer_key: an XML string of your answer key, for automatically - scored qualification tests. - (Consider implementing an AnswerKey class for this to support.) - - test_duration: the number of seconds a worker has to complete the test. - - auto_granted: if True, requests for the Qualification are granted immediately. - Can't coexist with a test. - - auto_granted_value: auto_granted qualifications are given this value. - - """ - - params = {'Name' : name, - 'Description' : description, - 'QualificationTypeStatus' : status, - } - if retry_delay is not None: - params['RetryDelay'] = retry_delay - - if test is not None: - assert(isinstance(test, QuestionForm)) - assert(test_duration is not None) - params['Test'] = test.get_as_xml() - - if test_duration is not None: - params['TestDuration'] = test_duration - - if answer_key is not None: - if isinstance(answer_key, basestring): - params['AnswerKey'] = answer_key # xml - else: - raise TypeError - # Eventually someone will write an AnswerKey class. - - if auto_granted: - assert(test is False) - params['AutoGranted'] = True - params['AutoGrantedValue'] = auto_granted_value - - if keywords: - params['Keywords'] = self.get_keywords_as_string(keywords) - - return self._process_request('CreateQualificationType', params, - [('QualificationType', QualificationType),]) - - def get_qualification_type(self, qualification_type_id): - params = {'QualificationTypeId' : qualification_type_id } - return self._process_request('GetQualificationType', params, - [('QualificationType', QualificationType),]) - - def get_qualifications_for_qualification_type(self, qualification_type_id): - params = {'QualificationTypeId' : qualification_type_id } - return self._process_request('GetQualificationsForQualificationType', params, - [('QualificationType', QualificationType),]) - - def update_qualification_type(self, qualification_type_id, - description=None, - status=None, - retry_delay=None, - test=None, - answer_key=None, - test_duration=None, - auto_granted=None, - auto_granted_value=None): - - params = {'QualificationTypeId' : qualification_type_id } - - if description is not None: - params['Description'] = description - - if status is not None: - params['QualificationTypeStatus'] = status - - if retry_delay is not None: - params['RetryDelay'] = retry_delay - - if test is not None: - assert(isinstance(test, QuestionForm)) - params['Test'] = test.get_as_xml() - - if test_duration is not None: - params['TestDuration'] = test_duration - - if answer_key is not None: - if isinstance(answer_key, basestring): - params['AnswerKey'] = answer_key # xml - else: - raise TypeError - # Eventually someone will write an AnswerKey class. - - if auto_granted is not None: - params['AutoGranted'] = auto_granted - - if auto_granted_value is not None: - params['AutoGrantedValue'] = auto_granted_value - - return self._process_request('UpdateQualificationType', params, - [('QualificationType', QualificationType),]) - - def dispose_qualification_type(self, qualification_type_id): - """TODO: Document.""" - params = {'QualificationTypeId' : qualification_type_id} - return self._process_request('DisposeQualificationType', params) - - def search_qualification_types(self, query=None, sort_by='Name', - sort_direction='Ascending', page_size=10, - page_number=1, must_be_requestable=True, - must_be_owned_by_caller=True): - """TODO: Document.""" - params = {'Query' : query, - 'SortProperty' : sort_by, - 'SortDirection' : sort_direction, - 'PageSize' : page_size, - 'PageNumber' : page_number, - 'MustBeRequestable' : must_be_requestable, - 'MustBeOwnedByCaller' : must_be_owned_by_caller} - return self._process_request('SearchQualificationTypes', params, - [('QualificationType', QualificationType),]) - - def get_qualification_requests(self, qualification_type_id, - sort_by='Expiration', - sort_direction='Ascending', page_size=10, - page_number=1): - """TODO: Document.""" - params = {'QualificationTypeId' : qualification_type_id, - 'SortProperty' : sort_by, - 'SortDirection' : sort_direction, - 'PageSize' : page_size, - 'PageNumber' : page_number} - return self._process_request('GetQualificationRequests', params, - [('QualificationRequest', QualificationRequest),]) - - def grant_qualification(self, qualification_request_id, integer_value=1): - """TODO: Document.""" - params = {'QualificationRequestId' : qualification_request_id, - 'IntegerValue' : integer_value} - return self._process_request('GrantQualification', params) - - def revoke_qualification(self, subject_id, qualification_type_id, - reason=None): - """TODO: Document.""" - params = {'SubjectId' : subject_id, - 'QualificationTypeId' : qualification_type_id, - 'Reason' : reason} - return self._process_request('RevokeQualification', params) - - def assign_qualification(self, qualification_type_id, worker_id, - value=1, send_notification=True): - params = {'QualificationTypeId' : qualification_type_id, - 'WorkerId' : worker_id, - 'IntegerValue' : value, - 'SendNotification' : send_notification, } - return self._process_request('AssignQualification', params) - - def _process_request(self, request_type, params, marker_elems=None): - """ - Helper to process the xml response from AWS - """ - response = self.make_request(request_type, params, verb='POST') - return self._process_response(response, marker_elems) - - def _process_response(self, response, marker_elems=None): - """ - Helper to process the xml response from AWS - """ - body = response.read() - #print body - if '' not in body: - rs = ResultSet(marker_elems) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise MTurkRequestError(response.status, response.reason, body) - - @staticmethod - def get_keywords_as_string(keywords): - """ - Returns a comma+space-separated string of keywords from either - a list or a string - """ - if type(keywords) is list: - keywords = ', '.join(keywords) - if type(keywords) is str: - final_keywords = keywords - elif type(keywords) is unicode: - final_keywords = keywords.encode('utf-8') - elif keywords is None: - final_keywords = "" - else: - raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords)) - return final_keywords - - @staticmethod - def get_price_as_price(reward): - """ - Returns a Price data structure from either a float or a Price - """ - if isinstance(reward, Price): - final_price = reward - else: - final_price = Price(reward) - return final_price - - @staticmethod - def duration_as_seconds(duration): - if isinstance(duration, datetime.timedelta): - duration = duration.days*86400 + duration.seconds - try: - duration = int(duration) - except TypeError: - raise TypeError("Duration must be a timedelta or int-castable, got %s" % type(duration)) - return duration - -class BaseAutoResultElement: - """ - Base class to automatically add attributes when parsing XML - """ - def __init__(self, connection): - pass - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - setattr(self, name, value) - -class HIT(BaseAutoResultElement): - """ - Class to extract a HIT structure from a response (used in ResultSet) - - Will have attributes named as per the Developer Guide, - e.g. HITId, HITTypeId, CreationTime - """ - - # property helper to determine if HIT has expired - def _has_expired(self): - """ Has this HIT expired yet? """ - expired = False - if hasattr(self, 'Expiration'): - now = datetime.datetime.utcnow() - expiration = datetime.datetime.strptime(self.Expiration, '%Y-%m-%dT%H:%M:%SZ') - expired = (now >= expiration) - else: - raise ValueError("ERROR: Request for expired property, but no Expiration in HIT!") - return expired - - # are we there yet? - expired = property(_has_expired) - -class QualificationType(BaseAutoResultElement): - """ - Class to extract an QualificationType structure from a response (used in - ResultSet) - - Will have attributes named as per the Developer Guide, - e.g. QualificationTypeId, CreationTime, Name, etc - """ - - pass - -class QualificationRequest(BaseAutoResultElement): - """ - Class to extract an QualificationRequest structure from a response (used in - ResultSet) - - Will have attributes named as per the Developer Guide, - e.g. QualificationRequestId, QualificationTypeId, SubjectId, etc - - TODO: Ensure that Test and Answer attribute are treated properly if the - qualification requires a test. These attributes are XML-encoded. - """ - - pass - -class Assignment(BaseAutoResultElement): - """ - Class to extract an Assignment structure from a response (used in - ResultSet) - - Will have attributes named as per the Developer Guide, - e.g. AssignmentId, WorkerId, HITId, Answer, etc - """ - - def __init__(self, connection): - BaseAutoResultElement.__init__(self, connection) - self.answers = [] - - def endElement(self, name, value, connection): - # the answer consists of embedded XML, so it needs to be parsed independantly - if name == 'Answer': - answer_rs = ResultSet([('Answer', QuestionFormAnswer),]) - h = handler.XmlHandler(answer_rs, connection) - value = connection.get_utf8_value(value) - xml.sax.parseString(value, h) - self.answers.append(answer_rs) - else: - BaseAutoResultElement.endElement(self, name, value, connection) - -class QuestionFormAnswer(BaseAutoResultElement): - """ - Class to extract Answers from inside the embedded XML - QuestionFormAnswers element inside the Answer element which is - part of the Assignment structure - - A QuestionFormAnswers element contains an Answer element for each - question in the HIT or Qualification test for which the Worker - provided an answer. Each Answer contains a QuestionIdentifier - element whose value corresponds to the QuestionIdentifier of a - Question in the QuestionForm. See the QuestionForm data structure - for more information about questions and answer specifications. - - If the question expects a free-text answer, the Answer element - contains a FreeText element. This element contains the Worker's - answer - - *NOTE* - currently really only supports free-text and selection answers - """ - - def __init__(self, connection): - BaseAutoResultElement.__init__(self, connection) - self.fields = [] - self.qid = None - - def endElement(self, name, value, connection): - if name == 'QuestionIdentifier': - self.qid = value - elif name in ['FreeText', 'SelectionIdentifier'] and self.qid: - self.fields.append((self.qid,value)) - elif name == 'Answer': - self.qid = None diff --git a/backup/src/boto/mturk/notification.py b/backup/src/boto/mturk/notification.py deleted file mode 100644 index 2aa99ca..0000000 --- a/backup/src/boto/mturk/notification.py +++ /dev/null @@ -1,94 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Provides NotificationMessage and Event classes, with utility methods, for -implementations of the Mechanical Turk Notification API. -""" - -import hmac -try: - from hashlib import sha1 as sha -except ImportError: - import sha -import base64 -import re - -class NotificationMessage: - - NOTIFICATION_WSDL = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurk/2006-05-05/AWSMechanicalTurkRequesterNotification.wsdl" - NOTIFICATION_VERSION = '2006-05-05' - - SERVICE_NAME = "AWSMechanicalTurkRequesterNotification" - OPERATION_NAME = "Notify" - - EVENT_PATTERN = r"Event\.(?P\d+)\.(?P\w+)" - EVENT_RE = re.compile(EVENT_PATTERN) - - def __init__(self, d): - """ - Constructor; expects parameter d to be a dict of string parameters from a REST transport notification message - """ - self.signature = d['Signature'] # vH6ZbE0NhkF/hfNyxz2OgmzXYKs= - self.timestamp = d['Timestamp'] # 2006-05-23T23:22:30Z - self.version = d['Version'] # 2006-05-05 - assert d['method'] == NotificationMessage.OPERATION_NAME, "Method should be '%s'" % NotificationMessage.OPERATION_NAME - - # Build Events - self.events = [] - events_dict = {} - if 'Event' in d: - # TurboGears surprised me by 'doing the right thing' and making { 'Event': { '1': { 'EventType': ... } } } etc. - events_dict = d['Event'] - else: - for k in d: - v = d[k] - if k.startswith('Event.'): - ed = NotificationMessage.EVENT_RE.search(k).groupdict() - n = int(ed['n']) - param = str(ed['param']) - if n not in events_dict: - events_dict[n] = {} - events_dict[n][param] = v - for n in events_dict: - self.events.append(Event(events_dict[n])) - - def verify(self, secret_key): - """ - Verifies the authenticity of a notification message. - """ - verification_input = NotificationMessage.SERVICE_NAME + NotificationMessage.OPERATION_NAME + self.timestamp - signature_calc = self._auth_handler.sign_string(verification_input) - return self.signature == signature_calc - -class Event: - def __init__(self, d): - self.event_type = d['EventType'] - self.event_time_str = d['EventTime'] - self.hit_type = d['HITTypeId'] - self.hit_id = d['HITId'] - if 'AssignmentId' in d: # Not present in all event types - self.assignment_id = d['AssignmentId'] - - #TODO: build self.event_time datetime from string self.event_time_str - - def __repr__(self): - return "" % (self.event_type, self.hit_id) diff --git a/backup/src/boto/mturk/price.py b/backup/src/boto/mturk/price.py deleted file mode 100644 index 3c88a96..0000000 --- a/backup/src/boto/mturk/price.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Price: - - def __init__(self, amount=0.0, currency_code='USD'): - self.amount = amount - self.currency_code = currency_code - self.formatted_price = '' - - def __repr__(self): - if self.formatted_price: - return self.formatted_price - else: - return str(self.amount) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Amount': - self.amount = float(value) - elif name == 'CurrencyCode': - self.currency_code = value - elif name == 'FormattedPrice': - self.formatted_price = value - - def get_as_params(self, label, ord=1): - return {'%s.%d.Amount'%(label, ord) : str(self.amount), - '%s.%d.CurrencyCode'%(label, ord) : self.currency_code} diff --git a/backup/src/boto/mturk/qualification.py b/backup/src/boto/mturk/qualification.py deleted file mode 100644 index 6b620ec..0000000 --- a/backup/src/boto/mturk/qualification.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2008 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Qualifications: - - def __init__(self, requirements=None): - if requirements == None: - requirements = [] - self.requirements = requirements - - def add(self, req): - self.requirements.append(req) - - def get_as_params(self): - params = {} - assert(len(self.requirements) <= 10) - for n, req in enumerate(self.requirements): - reqparams = req.get_as_params() - for rp in reqparams: - params['QualificationRequirement.%s.%s' % ((n+1),rp) ] = reqparams[rp] - return params - - -class Requirement(object): - """ - Representation of a single requirement - """ - - def __init__(self, qualification_type_id, comparator, integer_value=None, required_to_preview=False): - self.qualification_type_id = qualification_type_id - self.comparator = comparator - self.integer_value = integer_value - self.required_to_preview = required_to_preview - - def get_as_params(self): - params = { - "QualificationTypeId": self.qualification_type_id, - "Comparator": self.comparator, - } - if self.comparator != 'Exists' and self.integer_value is not None: - params['IntegerValue'] = self.integer_value - if self.required_to_preview: - params['RequiredToPreview'] = "true" - return params - -class PercentAssignmentsSubmittedRequirement(Requirement): - """ - The percentage of assignments the Worker has submitted, over all assignments the Worker has accepted. The value is an integer between 0 and 100. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="00000000000000000000", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class PercentAssignmentsAbandonedRequirement(Requirement): - """ - The percentage of assignments the Worker has abandoned (allowed the deadline to elapse), over all assignments the Worker has accepted. The value is an integer between 0 and 100. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="00000000000000000070", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class PercentAssignmentsReturnedRequirement(Requirement): - """ - The percentage of assignments the Worker has returned, over all assignments the Worker has accepted. The value is an integer between 0 and 100. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="000000000000000000E0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class PercentAssignmentsApprovedRequirement(Requirement): - """ - The percentage of assignments the Worker has submitted that were subsequently approved by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="000000000000000000L0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class PercentAssignmentsRejectedRequirement(Requirement): - """ - The percentage of assignments the Worker has submitted that were subsequently rejected by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="000000000000000000S0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class NumberHitsApprovedRequirement(Requirement): - """ - Specifies the total number of HITs submitted by a Worker that have been approved. The value is an integer greater than or equal to 0. - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="00000000000000000040", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) - -class LocaleRequirement(Requirement): - """ - A Qualification requirement based on the Worker's location. The Worker's location is specified by the Worker to Mechanical Turk when the Worker creates his account. - """ - - def __init__(self, comparator, locale, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="00000000000000000071", comparator=comparator, integer_value=None, required_to_preview=required_to_preview) - self.locale = locale - - def get_as_params(self): - params = { - "QualificationTypeId": self.qualification_type_id, - "Comparator": self.comparator, - 'LocaleValue.Country': self.locale, - } - if self.required_to_preview: - params['RequiredToPreview'] = "true" - return params - -class AdultRequirement(Requirement): - """ - Requires workers to acknowledge that they are over 18 and that they agree to work on potentially offensive content. The value type is boolean, 1 (required), 0 (not required, the default). - """ - - def __init__(self, comparator, integer_value, required_to_preview=False): - Requirement.__init__(self, qualification_type_id="00000000000000000060", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) diff --git a/backup/src/boto/mturk/question.py b/backup/src/boto/mturk/question.py deleted file mode 100644 index b1556ad..0000000 --- a/backup/src/boto/mturk/question.py +++ /dev/null @@ -1,396 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Question(object): - template = "%(items)s" - - def __init__(self, identifier, content, answer_spec, is_required=False, display_name=None): - # copy all of the parameters into object attributes - self.__dict__.update(vars()) - del self.self - - def get_as_params(self, label='Question'): - return { label : self.get_as_xml() } - - def get_as_xml(self): - items = [ - SimpleField('QuestionIdentifier', self.identifier), - SimpleField('IsRequired', str(self.is_required).lower()), - self.content, - self.answer_spec, - ] - if self.display_name is not None: - items.insert(1, SimpleField('DisplayName', self.display_name)) - items = ''.join(item.get_as_xml() for item in items) - return self.template % vars() - -try: - from lxml import etree - class ValidatingXML(object): - def validate(self): - import urllib2 - schema_src_file = urllib2.urlopen(self.schema_url) - schema_doc = etree.parse(schema_src_file) - schema = etree.XMLSchema(schema_doc) - doc = etree.fromstring(self.get_as_xml()) - schema.assertValid(doc) -except ImportError: - class ValidatingXML(object): - def validate(self): pass - - -class ExternalQuestion(ValidatingXML): - """ - An object for constructing an External Question. - """ - schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd" - template = '%%(external_url)s%%(frame_height)s' % vars() - - def __init__(self, external_url, frame_height): - self.external_url = external_url - self.frame_height = frame_height - - def get_as_params(self, label='ExternalQuestion'): - return { label : self.get_as_xml() } - - def get_as_xml(self): - return self.template % vars(self) - -class XMLTemplate: - def get_as_xml(self): - return self.template % vars(self) - -class SimpleField(object, XMLTemplate): - """ - A Simple name/value pair that can be easily rendered as XML. - - >>> SimpleField('Text', 'A text string').get_as_xml() - 'A text string' - """ - template = '<%(field)s>%(value)s' - - def __init__(self, field, value): - self.field = field - self.value = value - -class Binary(object, XMLTemplate): - template = """%(type)s%(subtype)s%(url)s%(alt_text)s""" - def __init__(self, type, subtype, url, alt_text): - self.__dict__.update(vars()) - del self.self - -class List(list): - """A bulleted list suitable for OrderedContent or Overview content""" - def get_as_xml(self): - items = ''.join('%s' % item for item in self) - return '%s' % items - -class Application(object): - template = "<%(class_)s>%(content)s" - parameter_template = "%(name)s%(value)s" - - def __init__(self, width, height, **parameters): - self.width = width - self.height = height - self.parameters = parameters - - def get_inner_content(self, content): - content.append_field('Width', self.width) - content.append_field('Height', self.height) - for name, value in self.parameters.items(): - value = self.parameter_template % vars() - content.append_field('ApplicationParameter', value) - - def get_as_xml(self): - content = OrderedContent() - self.get_inner_content(content) - content = content.get_as_xml() - class_ = self.__class__.__name__ - return self.template % vars() - -class JavaApplet(Application): - def __init__(self, path, filename, *args, **kwargs): - self.path = path - self.filename = filename - super(JavaApplet, self).__init__(*args, **kwargs) - - def get_inner_content(self, content): - content = OrderedContent() - content.append_field('AppletPath', self.path) - content.append_field('AppletFilename', self.filename) - super(JavaApplet, self).get_inner_content(content) - -class Flash(Application): - def __init__(self, url, *args, **kwargs): - self.url = url - super(Flash, self).__init__(*args, **kwargs) - - def get_inner_content(self, content): - content = OrderedContent() - content.append_field('FlashMovieURL', self.url) - super(Flash, self).get_inner_content(content) - -class FormattedContent(object, XMLTemplate): - schema_url = 'http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/FormattedContentXHTMLSubset.xsd' - template = '' - def __init__(self, content): - self.content = content - -class OrderedContent(list): - - def append_field(self, field, value): - self.append(SimpleField(field, value)) - - def get_as_xml(self): - return ''.join(item.get_as_xml() for item in self) - -class Overview(OrderedContent): - template = '%(content)s' - - def get_as_params(self, label='Overview'): - return { label : self.get_as_xml() } - - def get_as_xml(self): - content = super(Overview, self).get_as_xml() - return self.template % vars() - -class QuestionForm(ValidatingXML, list): - """ - From the AMT API docs: - - The top-most element of the QuestionForm data structure is a QuestionForm element. This - element contains optional Overview elements and one or more Question elements. There can be - any number of these two element types listed in any order. The following example structure has an - Overview element and a Question element followed by a second Overview element and Question - element--all within the same QuestionForm. - - - - [...] - - - [...] - - - [...] - - - [...] - - [...] - - - QuestionForm is implemented as a list, so to construct a - QuestionForm, simply append Questions and Overviews (with at least - one Question). - """ - schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" - xml_template = """%%(items)s""" % vars() - - def is_valid(self): - return ( - any(isinstance(item, Question) for item in self) - and - all(isinstance(item, (Question, Overview)) for item in self) - ) - - def get_as_xml(self): - assert self.is_valid(), "QuestionForm contains invalid elements" - items = ''.join(item.get_as_xml() for item in self) - return self.xml_template % vars() - -class QuestionContent(OrderedContent): - template = '%(content)s' - - def get_as_xml(self): - content = super(QuestionContent, self).get_as_xml() - return self.template % vars() - -class AnswerSpecification(object): - template = '%(spec)s' - - def __init__(self, spec): - self.spec = spec - - def get_as_xml(self): - spec = self.spec.get_as_xml() - return self.template % vars() - -class Constraints(OrderedContent): - template = '%(content)s' - - def get_as_xml(self): - content = super(Constraints, self).get_as_xml() - return self.template % vars() - -class Constraint(object): - def get_attributes(self): - pairs = zip(self.attribute_names, self.attribute_values) - attrs = ' '.join( - '%s="%d"' % (name,value) - for (name,value) in pairs - if value is not None - ) - return attrs - - def get_as_xml(self): - attrs = self.get_attributes() - return self.template % vars() - -class NumericConstraint(Constraint): - attribute_names = 'minValue', 'maxValue' - template = '' - - def __init__(self, min_value=None, max_value=None): - self.attribute_values = min_value, max_value - -class LengthConstraint(Constraint): - attribute_names = 'minLength', 'maxLength' - template = '' - - def __init__(self, min_length=None, max_length=None): - self.attribute_values = min_length, max_length - -class RegExConstraint(Constraint): - attribute_names = 'regex', 'errorText', 'flags' - template = '' - - def __init__(self, pattern, error_text=None, flags=None): - self.attribute_values = pattern, error_text, flags - -class NumberOfLinesSuggestion(object): - template = '%(num_lines)s' - - def __init__(self, num_lines=1): - self.num_lines = num_lines - - def get_as_xml(self): - num_lines = self.num_lines - return self.template % vars() - -class FreeTextAnswer(object): - template = '%(items)s' - - def __init__(self, default=None, constraints=None, num_lines=None): - self.default = default - if constraints is None: constraints = Constraints() - self.constraints = Constraints(constraints) - self.num_lines = num_lines - - def get_as_xml(self): - constraints = Constraints() - items = [constraints] - if self.default: - items.append(SimpleField('DefaultText', self.default)) - if self.num_lines: - items.append(NumberOfLinesSuggestion(self.num_lines)) - items = ''.join(item.get_as_xml() for item in items) - return self.template % vars() - -class FileUploadAnswer(object): - template = """%(min_bytes)d%(max_bytes)d""" - - def __init__(self, min_bytes, max_bytes): - assert 0 <= min_bytes <= max_bytes <= 2*10**9 - self.min_bytes = min_bytes - self.max_bytes = max_bytes - - def get_as_xml(self): - return self.template % vars(self) - -class SelectionAnswer(object): - """ - A class to generate SelectionAnswer XML data structures. - Does not yet implement Binary selection options. - """ - SELECTIONANSWER_XML_TEMPLATE = """%s%s%s""" # % (count_xml, style_xml, selections_xml) - SELECTION_XML_TEMPLATE = """%s%s""" # (identifier, value_xml) - SELECTION_VALUE_XML_TEMPLATE = """<%s>%s""" # (type, value, type) - STYLE_XML_TEMPLATE = """%s""" # (style) - MIN_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count - MAX_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count - ACCEPTED_STYLES = ['radiobutton', 'dropdown', 'checkbox', 'list', 'combobox', 'multichooser'] - OTHER_SELECTION_ELEMENT_NAME = 'OtherSelection' - - def __init__(self, min=1, max=1, style=None, selections=None, type='text', other=False): - - if style is not None: - if style in SelectionAnswer.ACCEPTED_STYLES: - self.style_suggestion = style - else: - raise ValueError("style '%s' not recognized; should be one of %s" % (style, ', '.join(SelectionAnswer.ACCEPTED_STYLES))) - else: - self.style_suggestion = None - - if selections is None: - raise ValueError("SelectionAnswer.__init__(): selections must be a non-empty list of (content, identifier) tuples") - else: - self.selections = selections - - self.min_selections = min - self.max_selections = max - - assert len(selections) >= self.min_selections, "# of selections is less than minimum of %d" % self.min_selections - #assert len(selections) <= self.max_selections, "# of selections exceeds maximum of %d" % self.max_selections - - self.type = type - - self.other = other - - def get_as_xml(self): - if self.type == 'text': - TYPE_TAG = "Text" - elif self.type == 'binary': - TYPE_TAG = "Binary" - else: - raise ValueError("illegal type: %s; must be either 'text' or 'binary'" % str(self.type)) - - # build list of elements - selections_xml = "" - for tpl in self.selections: - value_xml = SelectionAnswer.SELECTION_VALUE_XML_TEMPLATE % (TYPE_TAG, tpl[0], TYPE_TAG) - selection_xml = SelectionAnswer.SELECTION_XML_TEMPLATE % (tpl[1], value_xml) - selections_xml += selection_xml - - if self.other: - # add OtherSelection element as xml if available - if hasattr(self.other, 'get_as_xml'): - assert type(self.other) == FreeTextAnswer, 'OtherSelection can only be a FreeTextAnswer' - selections_xml += self.other.get_as_xml().replace('FreeTextAnswer', 'OtherSelection') - else: - selections_xml += "" - - if self.style_suggestion is not None: - style_xml = SelectionAnswer.STYLE_XML_TEMPLATE % self.style_suggestion - else: - style_xml = "" - - if self.style_suggestion != 'radiobutton': - count_xml = SelectionAnswer.MIN_SELECTION_COUNT_XML_TEMPLATE %self.min_selections - count_xml += SelectionAnswer.MAX_SELECTION_COUNT_XML_TEMPLATE %self.max_selections - else: - count_xml = "" - - ret = SelectionAnswer.SELECTIONANSWER_XML_TEMPLATE % (count_xml, style_xml, selections_xml) - - # return XML - return ret - diff --git a/backup/src/boto/mturk/test/.gitignore b/backup/src/boto/mturk/test/.gitignore deleted file mode 100644 index 039e4d4..0000000 --- a/backup/src/boto/mturk/test/.gitignore +++ /dev/null @@ -1 +0,0 @@ -local.py diff --git a/backup/src/boto/mturk/test/__init__.py b/backup/src/boto/mturk/test/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/backup/src/boto/mturk/test/_init_environment.py b/backup/src/boto/mturk/test/_init_environment.py deleted file mode 100644 index ec026ec..0000000 --- a/backup/src/boto/mturk/test/_init_environment.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import functools - -live_connection = False -mturk_host = 'mechanicalturk.sandbox.amazonaws.com' -external_url = 'http://www.example.com/' - -try: - local = os.path.join(os.path.dirname(__file__), 'local.py') - execfile(local) -except: - pass - -if live_connection: - #TODO: you must set the auth credentials to something valid - from boto.mturk.connection import MTurkConnection -else: - # Here the credentials must be set, but it doesn't matter what - # they're set to. - os.environ.setdefault('AWS_ACCESS_KEY_ID', 'foo') - os.environ.setdefault('AWS_SECRET_ACCESS_KEY', 'bar') - from mocks import MTurkConnection - -SetHostMTurkConnection = functools.partial(MTurkConnection, host=mturk_host) diff --git a/backup/src/boto/mturk/test/all_tests.py b/backup/src/boto/mturk/test/all_tests.py deleted file mode 100644 index f17cf85..0000000 --- a/backup/src/boto/mturk/test/all_tests.py +++ /dev/null @@ -1,24 +0,0 @@ - -import unittest -import doctest -from glob import glob - -from create_hit_test import * -from create_hit_with_qualifications import * -from create_hit_external import * -from create_hit_with_qualifications import * -from hit_persistence import * - -doctest_suite = doctest.DocFileSuite( - *glob('*.doctest'), - optionflags=doctest.REPORT_ONLY_FIRST_FAILURE - ) - -class Program(unittest.TestProgram): - def runTests(self, *args, **kwargs): - self.test = unittest.TestSuite([self.test, doctest_suite]) - super(Program, self).runTests(*args, **kwargs) - -if __name__ == '__main__': - Program() - diff --git a/backup/src/boto/mturk/test/cleanup_tests.py b/backup/src/boto/mturk/test/cleanup_tests.py deleted file mode 100644 index 2381dd9..0000000 --- a/backup/src/boto/mturk/test/cleanup_tests.py +++ /dev/null @@ -1,45 +0,0 @@ -import itertools - -from _init_environment import SetHostMTurkConnection - -def description_filter(substring): - return lambda hit: substring in hit.Title - -def disable_hit(hit): - return conn.disable_hit(hit.HITId) - -def dispose_hit(hit): - # assignments must be first approved or rejected - for assignment in conn.get_assignments(hit.HITId): - if assignment.AssignmentStatus == 'Submitted': - conn.approve_assignment(assignment.AssignmentId) - return conn.dispose_hit(hit.HITId) - -def cleanup(): - """Remove any boto test related HIT's""" - - global conn - - conn = SetHostMTurkConnection() - - - is_boto = description_filter('Boto') - print 'getting hits...' - all_hits = list(conn.get_all_hits()) - is_reviewable = lambda hit: hit.HITStatus == 'Reviewable' - is_not_reviewable = lambda hit: not is_reviewable(hit) - hits_to_process = filter(is_boto, all_hits) - hits_to_disable = filter(is_not_reviewable, hits_to_process) - hits_to_dispose = filter(is_reviewable, hits_to_process) - print 'disabling/disposing %d/%d hits' % (len(hits_to_disable), len(hits_to_dispose)) - map(disable_hit, hits_to_disable) - map(dispose_hit, hits_to_dispose) - - total_hits = len(all_hits) - hits_processed = len(hits_to_process) - skipped = total_hits - hits_processed - fmt = 'Processed: %(total_hits)d HITs, disabled/disposed: %(hits_processed)d, skipped: %(skipped)d' - print fmt % vars() - -if __name__ == '__main__': - cleanup() diff --git a/backup/src/boto/mturk/test/common.py b/backup/src/boto/mturk/test/common.py deleted file mode 100644 index 23361bd..0000000 --- a/backup/src/boto/mturk/test/common.py +++ /dev/null @@ -1,44 +0,0 @@ -import unittest -import uuid -import datetime - -from boto.mturk.question import ( - Question, QuestionContent, AnswerSpecification, FreeTextAnswer, -) -from _init_environment import SetHostMTurkConnection - -class MTurkCommon(unittest.TestCase): - def setUp(self): - self.conn = SetHostMTurkConnection() - - @staticmethod - def get_question(): - # create content for a question - qn_content = QuestionContent() - qn_content.append_field('Title', 'Boto no hit type question content') - qn_content.append_field('Text', 'What is a boto no hit type?') - - # create the question specification - qn = Question(identifier=str(uuid.uuid4()), - content=qn_content, - answer_spec=AnswerSpecification(FreeTextAnswer())) - return qn - - @staticmethod - def get_hit_params(): - return dict( - lifetime=datetime.timedelta(minutes=65), - max_assignments=2, - title='Boto create_hit title', - description='Boto create_hit description', - keywords=['boto', 'test'], - reward=0.23, - duration=datetime.timedelta(minutes=6), - approval_delay=60*60, - annotation='An annotation from boto create_hit test', - response_groups=['Minimal', - 'HITDetail', - 'HITQuestion', - 'HITAssignmentSummary',], - ) - diff --git a/backup/src/boto/mturk/test/create_free_text_question_regex.doctest b/backup/src/boto/mturk/test/create_free_text_question_regex.doctest deleted file mode 100644 index 0b9d2a9..0000000 --- a/backup/src/boto/mturk/test/create_free_text_question_regex.doctest +++ /dev/null @@ -1,100 +0,0 @@ ->>> import uuid ->>> import datetime ->>> from _init_environment import MTurkConnection, mturk_host ->>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, RegExConstraint - ->>> conn = MTurkConnection(host=mturk_host) - -# create content for a question ->>> qn_content = QuestionContent() ->>> qn_content.append_field('Title', 'Boto no hit type question content') ->>> qn_content.append_field('Text', 'What is a boto no hit type?') - -# create a free text answer that is not quite so free! ->>> constraints = [ -... RegExConstraint( -... "^[12][0-9]{3}-[01]?\d-[0-3]?\d$", -... error_text="You must enter a date with the format yyyy-mm-dd.", -... flags='i', -... )] ->>> ft_answer = FreeTextAnswer(constraints=constraints, -... default="This is not a valid format") - -# create the question specification ->>> qn = Question(identifier=str(uuid.uuid4()), -... content=qn_content, -... answer_spec=AnswerSpecification(ft_answer)) - -# now, create the actual HIT for the question without using a HIT type -# NOTE - the response_groups are specified to get back additional information for testing ->>> keywords=['boto', 'test', 'doctest'] ->>> create_hit_rs = conn.create_hit(question=qn, -... lifetime=60*65, -... max_assignments=2, -... title='Boto create_hit title', -... description='Boto create_hit description', -... keywords=keywords, -... reward=0.23, -... duration=60*6, -... approval_delay=60*60, -... annotation='An annotation from boto create_hit test', -... response_groups=['Minimal', -... 'HITDetail', -... 'HITQuestion', -... 'HITAssignmentSummary',]) - -# this is a valid request ->>> create_hit_rs.status -True - -# for the requested hit type id -# the HIT Type Id is a unicode string ->>> len(create_hit_rs) -1 ->>> hit = create_hit_rs[0] ->>> hit_type_id = hit.HITTypeId ->>> hit_type_id # doctest: +ELLIPSIS -u'...' - ->>> hit.MaxAssignments -u'2' - ->>> hit.AutoApprovalDelayInSeconds -u'3600' - -# expiration should be very close to now + the lifetime in seconds ->>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) ->>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') ->>> delta = expected_datetime - expiration_datetime ->>> abs(delta).seconds < 5 -True - -# duration is as specified for the HIT type ->>> hit.AssignmentDurationInSeconds -u'360' - -# the reward has been set correctly (allow for float error here) ->>> int(float(hit.Amount) * 100) -23 - ->>> hit.FormattedPrice -u'$0.23' - -# only US currency supported at present ->>> hit.CurrencyCode -u'USD' - -# title is the HIT type title ->>> hit.Title -u'Boto create_hit title' - -# title is the HIT type description ->>> hit.Description -u'Boto create_hit description' - -# annotation is correct ->>> hit.RequesterAnnotation -u'An annotation from boto create_hit test' - ->>> hit.HITReviewStatus -u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit.doctest b/backup/src/boto/mturk/test/create_hit.doctest deleted file mode 100644 index a97cbf8..0000000 --- a/backup/src/boto/mturk/test/create_hit.doctest +++ /dev/null @@ -1,92 +0,0 @@ ->>> import uuid ->>> import datetime ->>> from _init_environment import MTurkConnection, mturk_host ->>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer - ->>> conn = MTurkConnection(host=mturk_host) - -# create content for a question ->>> qn_content = QuestionContent() ->>> qn_content.append_field('Title', 'Boto no hit type question content') ->>> qn_content.append_field('Text', 'What is a boto no hit type?') - -# create the question specification ->>> qn = Question(identifier=str(uuid.uuid4()), -... content=qn_content, -... answer_spec=AnswerSpecification(FreeTextAnswer())) - -# now, create the actual HIT for the question without using a HIT type -# NOTE - the response_groups are specified to get back additional information for testing ->>> keywords=['boto', 'test', 'doctest'] ->>> lifetime = datetime.timedelta(minutes=65) ->>> create_hit_rs = conn.create_hit(question=qn, -... lifetime=lifetime, -... max_assignments=2, -... title='Boto create_hit title', -... description='Boto create_hit description', -... keywords=keywords, -... reward=0.23, -... duration=60*6, -... approval_delay=60*60, -... annotation='An annotation from boto create_hit test', -... response_groups=['Minimal', -... 'HITDetail', -... 'HITQuestion', -... 'HITAssignmentSummary',]) - -# this is a valid request ->>> create_hit_rs.status -True - ->>> len(create_hit_rs) -1 ->>> hit = create_hit_rs[0] - -# for the requested hit type id -# the HIT Type Id is a unicode string ->>> hit_type_id = hit.HITTypeId ->>> hit_type_id # doctest: +ELLIPSIS -u'...' - ->>> hit.MaxAssignments -u'2' - ->>> hit.AutoApprovalDelayInSeconds -u'3600' - -# expiration should be very close to now + the lifetime ->>> expected_datetime = datetime.datetime.utcnow() + lifetime ->>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') ->>> delta = expected_datetime - expiration_datetime ->>> abs(delta).seconds < 5 -True - -# duration is as specified for the HIT type ->>> hit.AssignmentDurationInSeconds -u'360' - -# the reward has been set correctly (allow for float error here) ->>> int(float(hit.Amount) * 100) -23 - ->>> hit.FormattedPrice -u'$0.23' - -# only US currency supported at present ->>> hit.CurrencyCode -u'USD' - -# title is the HIT type title ->>> hit.Title -u'Boto create_hit title' - -# title is the HIT type description ->>> hit.Description -u'Boto create_hit description' - -# annotation is correct ->>> hit.RequesterAnnotation -u'An annotation from boto create_hit test' - ->>> hit.HITReviewStatus -u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_binary.doctest b/backup/src/boto/mturk/test/create_hit_binary.doctest deleted file mode 100644 index 3f0434e..0000000 --- a/backup/src/boto/mturk/test/create_hit_binary.doctest +++ /dev/null @@ -1,94 +0,0 @@ ->>> import uuid ->>> import datetime ->>> from _init_environment import MTurkConnection, mturk_host ->>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, Binary - ->>> conn = MTurkConnection(host=mturk_host) - -# create content for a question ->>> qn_content = QuestionContent() ->>> qn_content.append_field('Title','Boto no hit type question content') ->>> qn_content.append_field('Text', 'What is a boto binary hit type?') ->>> binary_content = Binary('image', 'jpeg', 'http://www.example.com/test1.jpg', alt_text='image is missing') ->>> qn_content.append(binary_content) - -# create the question specification ->>> qn = Question(identifier=str(uuid.uuid4()), -... content=qn_content, -... answer_spec=AnswerSpecification(FreeTextAnswer())) - -# now, create the actual HIT for the question without using a HIT type -# NOTE - the response_groups are specified to get back additional information for testing ->>> keywords=['boto', 'test', 'doctest'] ->>> lifetime = datetime.timedelta(minutes=65) ->>> create_hit_rs = conn.create_hit(question=qn, -... lifetime=lifetime, -... max_assignments=2, -... title='Boto create_hit title', -... description='Boto create_hit description', -... keywords=keywords, -... reward=0.23, -... duration=60*6, -... approval_delay=60*60, -... annotation='An annotation from boto create_hit test', -... response_groups=['Minimal', -... 'HITDetail', -... 'HITQuestion', -... 'HITAssignmentSummary',]) - -# this is a valid request ->>> create_hit_rs.status -True - ->>> len(create_hit_rs) -1 ->>> hit = create_hit_rs[0] - -# for the requested hit type id -# the HIT Type Id is a unicode string ->>> hit_type_id = hit.HITTypeId ->>> hit_type_id # doctest: +ELLIPSIS -u'...' - ->>> hit.MaxAssignments -u'2' - ->>> hit.AutoApprovalDelayInSeconds -u'3600' - -# expiration should be very close to now + the lifetime ->>> expected_datetime = datetime.datetime.utcnow() + lifetime ->>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') ->>> delta = expected_datetime - expiration_datetime ->>> abs(delta).seconds < 5 -True - -# duration is as specified for the HIT type ->>> hit.AssignmentDurationInSeconds -u'360' - -# the reward has been set correctly (allow for float error here) ->>> int(float(hit.Amount) * 100) -23 - ->>> hit.FormattedPrice -u'$0.23' - -# only US currency supported at present ->>> hit.CurrencyCode -u'USD' - -# title is the HIT type title ->>> hit.Title -u'Boto create_hit title' - -# title is the HIT type description ->>> hit.Description -u'Boto create_hit description' - -# annotation is correct ->>> hit.RequesterAnnotation -u'An annotation from boto create_hit test' - ->>> hit.HITReviewStatus -u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_external.py b/backup/src/boto/mturk/test/create_hit_external.py deleted file mode 100644 index 9e955a6..0000000 --- a/backup/src/boto/mturk/test/create_hit_external.py +++ /dev/null @@ -1,17 +0,0 @@ -import unittest -import uuid -import datetime -from boto.mturk.question import ExternalQuestion - -from _init_environment import SetHostMTurkConnection, external_url - -class Test(unittest.TestCase): - def test_create_hit_external(self): - q = ExternalQuestion(external_url=external_url, frame_height=800) - conn = SetHostMTurkConnection() - keywords=['boto', 'test', 'doctest'] - create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',]) - assert(create_hit_rs.status == True) - -if __name__ == "__main__": - unittest.main() diff --git a/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest b/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest deleted file mode 100644 index 1b6d0f0..0000000 --- a/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest +++ /dev/null @@ -1,103 +0,0 @@ ->>> import uuid ->>> import datetime ->>> from _init_environment import MTurkConnection, mturk_host ->>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer ->>> ->>> conn = MTurkConnection(host=mturk_host) ->>> keywords=['boto', 'test', 'doctest'] ->>> hit_type_rs = conn.register_hit_type('Boto Test HIT type', -... 'HIT Type for testing Boto', -... 0.12, -... 60*6, -... keywords=keywords, -... approval_delay=60*60) - -# this was a valid request ->>> hit_type_rs.status -True - -# the HIT Type Id is a unicode string ->>> hit_type_id = hit_type_rs.HITTypeId ->>> hit_type_id # doctest: +ELLIPSIS -u'...' - -# create content for a question ->>> qn_content = QuestionContent() ->>> qn_content.append_field('Title', 'Boto question content create_hit_from_hit_type') ->>> qn_content.append_field('Text', 'What is a boto create_hit_from_hit_type?') - -# create the question specification ->>> qn = Question(identifier=str(uuid.uuid4()), -... content=qn_content, -... answer_spec=AnswerSpecification(FreeTextAnswer())) - -# now, create the actual HIT for the question using the HIT type -# NOTE - the response_groups are specified to get back additional information for testing ->>> create_hit_rs = conn.create_hit(hit_type=hit_type_rs.HITTypeId, -... question=qn, -... lifetime=60*65, -... max_assignments=2, -... annotation='An annotation from boto create_hit_from_hit_type test', -... response_groups=['Minimal', -... 'HITDetail', -... 'HITQuestion', -... 'HITAssignmentSummary',]) - -# this is a valid request ->>> create_hit_rs.status -True - ->>> len(create_hit_rs) -1 - ->>> hit = create_hit_rs[0] - -# for the requested hit type id ->>> hit.HITTypeId == hit_type_id -True - -# with the correct number of maximum assignments ->>> hit.MaxAssignments -u'2' - -# and the approval delay ->>> hit.AutoApprovalDelayInSeconds -u'3600' - -# expiration should be very close to now + the lifetime in seconds ->>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) ->>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') ->>> delta = expected_datetime - expiration_datetime ->>> abs(delta).seconds < 5 -True - -# duration is as specified for the HIT type ->>> hit.AssignmentDurationInSeconds -u'360' - -# the reward has been set correctly ->>> float(hit.Amount) == 0.12 -True - ->>> hit.FormattedPrice -u'$0.12' - -# only US currency supported at present ->>> hit.CurrencyCode -u'USD' - -# title is the HIT type title ->>> hit.Title -u'Boto Test HIT type' - -# title is the HIT type description ->>> hit.Description -u'HIT Type for testing Boto' - -# annotation is correct ->>> hit.RequesterAnnotation -u'An annotation from boto create_hit_from_hit_type test' - -# not reviewed yet ->>> hit.HITReviewStatus -u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_test.py b/backup/src/boto/mturk/test/create_hit_test.py deleted file mode 100644 index a690d80..0000000 --- a/backup/src/boto/mturk/test/create_hit_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -import os -from boto.mturk.question import QuestionForm - -from common import MTurkCommon - -class TestHITCreation(MTurkCommon): - def testCallCreateHitWithOneQuestion(self): - create_hit_rs = self.conn.create_hit( - question=self.get_question(), - **self.get_hit_params() - ) - - def testCallCreateHitWithQuestionForm(self): - create_hit_rs = self.conn.create_hit( - questions=QuestionForm([self.get_question()]), - **self.get_hit_params() - ) - -if __name__ == '__main__': - unittest.main() diff --git a/backup/src/boto/mturk/test/create_hit_with_qualifications.py b/backup/src/boto/mturk/test/create_hit_with_qualifications.py deleted file mode 100644 index 9ef2bc5..0000000 --- a/backup/src/boto/mturk/test/create_hit_with_qualifications.py +++ /dev/null @@ -1,16 +0,0 @@ -from boto.mturk.connection import MTurkConnection -from boto.mturk.question import ExternalQuestion -from boto.mturk.qualification import Qualifications, PercentAssignmentsApprovedRequirement - -def test(): - q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) - conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') - keywords=['boto', 'test', 'doctest'] - qualifications = Qualifications() - qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")) - create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications) - assert(create_hit_rs.status == True) - print create_hit_rs.HITTypeId - -if __name__ == "__main__": - test() diff --git a/backup/src/boto/mturk/test/hit_persistence.py b/backup/src/boto/mturk/test/hit_persistence.py deleted file mode 100644 index 6991856..0000000 --- a/backup/src/boto/mturk/test/hit_persistence.py +++ /dev/null @@ -1,27 +0,0 @@ -import unittest -import pickle - -from common import MTurkCommon - -class TestHITPersistence(MTurkCommon): - def create_hit_result(self): - return self.conn.create_hit( - question=self.get_question(), **self.get_hit_params() - ) - - def test_pickle_hit_result(self): - result = self.create_hit_result() - new_result = pickle.loads(pickle.dumps(result)) - - def test_pickle_deserialized_version(self): - """ - It seems the technique used to store and reload the object must - result in an equivalent object, or subsequent pickles may fail. - This tests a double-pickle to elicit that error. - """ - result = self.create_hit_result() - new_result = pickle.loads(pickle.dumps(result)) - pickle.dumps(new_result) - -if __name__ == '__main__': - unittest.main() diff --git a/backup/src/boto/mturk/test/mocks.py b/backup/src/boto/mturk/test/mocks.py deleted file mode 100644 index d3f0f2e..0000000 --- a/backup/src/boto/mturk/test/mocks.py +++ /dev/null @@ -1,11 +0,0 @@ -from boto.mturk.connection import MTurkConnection as RealMTurkConnection - -class MTurkConnection(RealMTurkConnection): - """ - Mock MTurkConnection that doesn't connect, but instead just prepares - the request and captures information about its usage. - """ - - def _process_request(self, *args, **kwargs): - saved_args = self.__dict__.setdefault('_mock_saved_args', dict()) - saved_args['_process_request'] = (args, kwargs) diff --git a/backup/src/boto/mturk/test/reviewable_hits.doctest b/backup/src/boto/mturk/test/reviewable_hits.doctest deleted file mode 100644 index 113a056..0000000 --- a/backup/src/boto/mturk/test/reviewable_hits.doctest +++ /dev/null @@ -1,129 +0,0 @@ ->>> import uuid ->>> import datetime ->>> from _init_environment import MTurkConnection, mturk_host ->>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer - ->>> conn = MTurkConnection(host=mturk_host) - -# create content for a question ->>> qn_content = QuestionContent() ->>> qn_content.append_field('Title', 'Boto no hit type question content') ->>> qn_content.append_field('Text', 'What is a boto no hit type?') - -# create the question specification ->>> qn = Question(identifier=str(uuid.uuid4()), -... content=qn_content, -... answer_spec=AnswerSpecification(FreeTextAnswer())) - -# now, create the actual HIT for the question without using a HIT type -# NOTE - the response_groups are specified to get back additional information for testing ->>> keywords=['boto', 'test', 'doctest'] ->>> create_hit_rs = conn.create_hit(question=qn, -... lifetime=60*65, -... max_assignments=1, -... title='Boto Hit to be Reviewed', -... description='Boto reviewable_hits description', -... keywords=keywords, -... reward=0.23, -... duration=60*6, -... approval_delay=60*60, -... annotation='An annotation from boto create_hit test', -... response_groups=['Minimal', -... 'HITDetail', -... 'HITQuestion', -... 'HITAssignmentSummary',]) - -# this is a valid request ->>> create_hit_rs.status -True - ->>> len(create_hit_rs) -1 ->>> hit = create_hit_rs[0] - -# for the requested hit type id -# the HIT Type Id is a unicode string ->>> hit_type_id = hit.HITTypeId ->>> hit_type_id # doctest: +ELLIPSIS -u'...' - ->>> from selenium_support import complete_hit, has_selenium ->>> if has_selenium(): complete_hit(hit_type_id, response='reviewable_hits_test') ->>> import time - -Give mechanical turk some time to process the hit ->>> if has_selenium(): time.sleep(10) - -# should have some reviewable HIT's returned, especially if returning all HIT type's -# NOTE: but only if your account has existing HIT's in the reviewable state ->>> reviewable_rs = conn.get_reviewable_hits() - -# this is a valid request ->>> reviewable_rs.status -True - ->>> len(reviewable_rs) >= 1 -True - -# should contain at least one HIT object ->>> reviewable_rs # doctest: +ELLIPSIS -[>> hit_id = reviewable_rs[0].HITId - -# check that we can retrieve the assignments for a HIT ->>> assignments_rs = conn.get_assignments(hit_id) - -# this is a valid request ->>> assignments_rs.status -True - ->>> int(assignments_rs.NumResults) >= 1 -True - ->>> len(assignments_rs) == int(assignments_rs.NumResults) -True - ->>> assignments_rs.PageNumber -u'1' - ->>> assignments_rs.TotalNumResults >= 1 -True - -# should contain at least one Assignment object ->>> assignments_rs # doctest: +ELLIPSIS -[>> assignment = assignments_rs[0] - ->>> assignment.HITId == hit_id -True - -# should have a valid status ->>> assignment.AssignmentStatus in ['Submitted', 'Approved', 'Rejected'] -True - -# should have returned at least one answer ->>> len(assignment.answers) > 0 -True - -# should contain at least one set of QuestionFormAnswer objects ->>> assignment.answers # doctest: +ELLIPSIS -[[>> answer = assignment.answers[0][0] - -# the answer should have exactly one field ->>> len(answer.fields) -1 - ->>> qid, text = answer.fields[0] - ->>> text # doctest: +ELLIPSIS -u'...' - -# question identifier should be a unicode string ->>> qid # doctest: +ELLIPSIS -u'...' - diff --git a/backup/src/boto/mturk/test/run-doctest.py b/backup/src/boto/mturk/test/run-doctest.py deleted file mode 100644 index f10c762..0000000 --- a/backup/src/boto/mturk/test/run-doctest.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import print_function - -import argparse -import doctest - -parser = argparse.ArgumentParser( - description="Run a test by name" - ) -parser.add_argument('test_name') -args = parser.parse_args() - -doctest.testfile( - args.test_name, - optionflags=doctest.REPORT_ONLY_FIRST_FAILURE - ) \ No newline at end of file diff --git a/backup/src/boto/mturk/test/search_hits.doctest b/backup/src/boto/mturk/test/search_hits.doctest deleted file mode 100644 index a79bab7..0000000 --- a/backup/src/boto/mturk/test/search_hits.doctest +++ /dev/null @@ -1,16 +0,0 @@ ->>> from _init_environment import MTurkConnection, mturk_host ->>> conn = MTurkConnection(host=mturk_host) - -# should have some HIT's returned by a search (but only if your account has existing HIT's) ->>> search_rs = conn.search_hits() - -# this is a valid request ->>> search_rs.status -True - ->>> len(search_rs) > 1 -True - ->>> search_rs # doctest: +ELLIPSIS -[= (2,7): - import unittest -else: - import unittest2 as unittest diff --git a/backup/src/boto/mturk/test/test_disable_hit.py b/backup/src/boto/mturk/test/test_disable_hit.py deleted file mode 100644 index 2e2701d..0000000 --- a/backup/src/boto/mturk/test/test_disable_hit.py +++ /dev/null @@ -1,11 +0,0 @@ -from boto.mturk.test.support import unittest - -from common import MTurkCommon -from boto.mturk.connection import MTurkRequestError - -class TestDisableHITs(MTurkCommon): - def test_disable_invalid_hit(self): - self.assertRaises(MTurkRequestError, self.conn.disable_hit, 'foo') - -if __name__ == '__main__': - unittest.main() diff --git a/backup/src/boto/plugin.py b/backup/src/boto/plugin.py deleted file mode 100644 index f8b592c..0000000 --- a/backup/src/boto/plugin.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -""" -Implements plugin related api. - -To define a new plugin just subclass Plugin, like this. - -class AuthPlugin(Plugin): - pass - -Then start creating subclasses of your new plugin. - -class MyFancyAuth(AuthPlugin): - capability = ['sign', 'vmac'] - -The actual interface is duck typed. - -""" - -import glob -import imp, os.path - -class Plugin(object): - """Base class for all plugins.""" - - capability = [] - - @classmethod - def is_capable(cls, requested_capability): - """Returns true if the requested capability is supported by this plugin - """ - for c in requested_capability: - if not c in cls.capability: - return False - return True - -def get_plugin(cls, requested_capability=None): - if not requested_capability: - requested_capability = [] - result = [] - for handler in cls.__subclasses__(): - if handler.is_capable(requested_capability): - result.append(handler) - return result - -def _import_module(filename): - (path, name) = os.path.split(filename) - (name, ext) = os.path.splitext(name) - - (file, filename, data) = imp.find_module(name, [path]) - try: - return imp.load_module(name, file, filename, data) - finally: - if file: - file.close() - -_plugin_loaded = False - -def load_plugins(config): - global _plugin_loaded - if _plugin_loaded: - return - _plugin_loaded = True - - if not config.has_option('Plugin', 'plugin_directory'): - return - directory = config.get('Plugin', 'plugin_directory') - for file in glob.glob(os.path.join(directory, '*.py')): - _import_module(file) - diff --git a/backup/src/boto/provider.py b/backup/src/boto/provider.py deleted file mode 100644 index c1c8b59..0000000 --- a/backup/src/boto/provider.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright 2010 Google Inc. -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -""" -This class encapsulates the provider-specific header differences. -""" - -import os -import boto -from boto import config -from boto.gs.acl import ACL -from boto.gs.acl import CannedACLStrings as CannedGSACLStrings -from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings -from boto.s3.acl import Policy - -HEADER_PREFIX_KEY = 'header_prefix' -METADATA_PREFIX_KEY = 'metadata_prefix' - -AWS_HEADER_PREFIX = 'x-amz-' -GOOG_HEADER_PREFIX = 'x-goog-' - -ACL_HEADER_KEY = 'acl-header' -AUTH_HEADER_KEY = 'auth-header' -COPY_SOURCE_HEADER_KEY = 'copy-source-header' -COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header' -DELETE_MARKER_HEADER_KEY = 'delete-marker-header' -DATE_HEADER_KEY = 'date-header' -METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header' -RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header' -SECURITY_TOKEN_HEADER_KEY = 'security-token-header' -STORAGE_CLASS_HEADER_KEY = 'storage-class' -MFA_HEADER_KEY = 'mfa-header' -VERSION_ID_HEADER_KEY = 'version-id-header' - -STORAGE_COPY_ERROR = 'StorageCopyError' -STORAGE_CREATE_ERROR = 'StorageCreateError' -STORAGE_DATA_ERROR = 'StorageDataError' -STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError' -STORAGE_RESPONSE_ERROR = 'StorageResponseError' - - -class Provider(object): - - CredentialMap = { - 'aws' : ('aws_access_key_id', 'aws_secret_access_key'), - 'google' : ('gs_access_key_id', 'gs_secret_access_key'), - } - - AclClassMap = { - 'aws' : Policy, - 'google' : ACL - } - - CannedAclsMap = { - 'aws' : CannedS3ACLStrings, - 'google' : CannedGSACLStrings - } - - HostKeyMap = { - 'aws' : 's3', - 'google' : 'gs' - } - - HeaderInfoMap = { - 'aws' : { - HEADER_PREFIX_KEY : AWS_HEADER_PREFIX, - METADATA_PREFIX_KEY : AWS_HEADER_PREFIX + 'meta-', - ACL_HEADER_KEY : AWS_HEADER_PREFIX + 'acl', - AUTH_HEADER_KEY : 'AWS', - COPY_SOURCE_HEADER_KEY : AWS_HEADER_PREFIX + 'copy-source', - COPY_SOURCE_VERSION_ID_HEADER_KEY : AWS_HEADER_PREFIX + - 'copy-source-version-id', - DATE_HEADER_KEY : AWS_HEADER_PREFIX + 'date', - DELETE_MARKER_HEADER_KEY : AWS_HEADER_PREFIX + 'delete-marker', - METADATA_DIRECTIVE_HEADER_KEY : AWS_HEADER_PREFIX + - 'metadata-directive', - RESUMABLE_UPLOAD_HEADER_KEY : None, - SECURITY_TOKEN_HEADER_KEY : AWS_HEADER_PREFIX + 'security-token', - VERSION_ID_HEADER_KEY : AWS_HEADER_PREFIX + 'version-id', - STORAGE_CLASS_HEADER_KEY : AWS_HEADER_PREFIX + 'storage-class', - MFA_HEADER_KEY : AWS_HEADER_PREFIX + 'mfa', - }, - 'google' : { - HEADER_PREFIX_KEY : GOOG_HEADER_PREFIX, - METADATA_PREFIX_KEY : GOOG_HEADER_PREFIX + 'meta-', - ACL_HEADER_KEY : GOOG_HEADER_PREFIX + 'acl', - AUTH_HEADER_KEY : 'GOOG1', - COPY_SOURCE_HEADER_KEY : GOOG_HEADER_PREFIX + 'copy-source', - COPY_SOURCE_VERSION_ID_HEADER_KEY : GOOG_HEADER_PREFIX + - 'copy-source-version-id', - DATE_HEADER_KEY : GOOG_HEADER_PREFIX + 'date', - DELETE_MARKER_HEADER_KEY : GOOG_HEADER_PREFIX + 'delete-marker', - METADATA_DIRECTIVE_HEADER_KEY : GOOG_HEADER_PREFIX + - 'metadata-directive', - RESUMABLE_UPLOAD_HEADER_KEY : GOOG_HEADER_PREFIX + 'resumable', - SECURITY_TOKEN_HEADER_KEY : GOOG_HEADER_PREFIX + 'security-token', - VERSION_ID_HEADER_KEY : GOOG_HEADER_PREFIX + 'version-id', - STORAGE_CLASS_HEADER_KEY : None, - MFA_HEADER_KEY : None, - } - } - - ErrorMap = { - 'aws' : { - STORAGE_COPY_ERROR : boto.exception.S3CopyError, - STORAGE_CREATE_ERROR : boto.exception.S3CreateError, - STORAGE_DATA_ERROR : boto.exception.S3DataError, - STORAGE_PERMISSIONS_ERROR : boto.exception.S3PermissionsError, - STORAGE_RESPONSE_ERROR : boto.exception.S3ResponseError, - }, - 'google' : { - STORAGE_COPY_ERROR : boto.exception.GSCopyError, - STORAGE_CREATE_ERROR : boto.exception.GSCreateError, - STORAGE_DATA_ERROR : boto.exception.GSDataError, - STORAGE_PERMISSIONS_ERROR : boto.exception.GSPermissionsError, - STORAGE_RESPONSE_ERROR : boto.exception.GSResponseError, - } - } - - def __init__(self, name, access_key=None, secret_key=None): - self.host = None - self.access_key = access_key - self.secret_key = secret_key - self.name = name - self.acl_class = self.AclClassMap[self.name] - self.canned_acls = self.CannedAclsMap[self.name] - self.get_credentials(access_key, secret_key) - self.configure_headers() - self.configure_errors() - # allow config file to override default host - host_opt_name = '%s_host' % self.HostKeyMap[self.name] - if config.has_option('Credentials', host_opt_name): - self.host = config.get('Credentials', host_opt_name) - - def get_credentials(self, access_key=None, secret_key=None): - access_key_name, secret_key_name = self.CredentialMap[self.name] - if access_key is not None: - self.access_key = access_key - elif os.environ.has_key(access_key_name.upper()): - self.access_key = os.environ[access_key_name.upper()] - elif config.has_option('Credentials', access_key_name): - self.access_key = config.get('Credentials', access_key_name) - - if secret_key is not None: - self.secret_key = secret_key - elif os.environ.has_key(secret_key_name.upper()): - self.secret_key = os.environ[secret_key_name.upper()] - elif config.has_option('Credentials', secret_key_name): - self.secret_key = config.get('Credentials', secret_key_name) - if isinstance(self.secret_key, unicode): - # the secret key must be bytes and not unicode to work - # properly with hmac.new (see http://bugs.python.org/issue5285) - self.secret_key = str(self.secret_key) - - def configure_headers(self): - header_info_map = self.HeaderInfoMap[self.name] - self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY] - self.header_prefix = header_info_map[HEADER_PREFIX_KEY] - self.acl_header = header_info_map[ACL_HEADER_KEY] - self.auth_header = header_info_map[AUTH_HEADER_KEY] - self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY] - self.copy_source_version_id = header_info_map[ - COPY_SOURCE_VERSION_ID_HEADER_KEY] - self.date_header = header_info_map[DATE_HEADER_KEY] - self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY] - self.metadata_directive_header = ( - header_info_map[METADATA_DIRECTIVE_HEADER_KEY]) - self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY] - self.resumable_upload_header = ( - header_info_map[RESUMABLE_UPLOAD_HEADER_KEY]) - self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY] - self.version_id = header_info_map[VERSION_ID_HEADER_KEY] - self.mfa_header = header_info_map[MFA_HEADER_KEY] - - def configure_errors(self): - error_map = self.ErrorMap[self.name] - self.storage_copy_error = error_map[STORAGE_COPY_ERROR] - self.storage_create_error = error_map[STORAGE_CREATE_ERROR] - self.storage_data_error = error_map[STORAGE_DATA_ERROR] - self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR] - self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR] - - def get_provider_name(self): - return self.HostKeyMap[self.name] - -# Static utility method for getting default Provider. -def get_default(): - return Provider('aws') diff --git a/backup/src/boto/pyami/__init__.py b/backup/src/boto/pyami/__init__.py deleted file mode 100644 index 303dbb6..0000000 --- a/backup/src/boto/pyami/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/backup/src/boto/pyami/bootstrap.py b/backup/src/boto/pyami/bootstrap.py deleted file mode 100644 index c1441fd..0000000 --- a/backup/src/boto/pyami/bootstrap.py +++ /dev/null @@ -1,125 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import os -import boto -from boto.utils import get_instance_metadata, get_instance_userdata -from boto.pyami.config import Config, BotoConfigPath -from boto.pyami.scriptbase import ScriptBase - -class Bootstrap(ScriptBase): - """ - The Bootstrap class is instantiated and run as part of the PyAMI - instance initialization process. The methods in this class will - be run from the rc.local script of the instance and will be run - as the root user. - - The main purpose of this class is to make sure the boto distribution - on the instance is the one required. - """ - - def __init__(self): - self.working_dir = '/mnt/pyami' - self.write_metadata() - ScriptBase.__init__(self) - - def write_metadata(self): - fp = open(os.path.expanduser(BotoConfigPath), 'w') - fp.write('[Instance]\n') - inst_data = get_instance_metadata() - for key in inst_data: - fp.write('%s = %s\n' % (key, inst_data[key])) - user_data = get_instance_userdata() - fp.write('\n%s\n' % user_data) - fp.write('[Pyami]\n') - fp.write('working_dir = %s\n' % self.working_dir) - fp.close() - # This file has the AWS credentials, should we lock it down? - # os.chmod(BotoConfigPath, stat.S_IREAD | stat.S_IWRITE) - # now that we have written the file, read it into a pyami Config object - boto.config = Config() - boto.init_logging() - - def create_working_dir(self): - boto.log.info('Working directory: %s' % self.working_dir) - if not os.path.exists(self.working_dir): - os.mkdir(self.working_dir) - - def load_boto(self): - update = boto.config.get('Boto', 'boto_update', 'svn:HEAD') - if update.startswith('svn'): - if update.find(':') >= 0: - method, version = update.split(':') - version = '-r%s' % version - else: - version = '-rHEAD' - location = boto.config.get('Boto', 'boto_location', '/usr/local/boto') - self.run('svn update %s %s' % (version, location)) - elif update.startswith('git'): - location = boto.config.get('Boto', 'boto_location', '/usr/share/python-support/python-boto/boto') - self.run('git pull', cwd=location) - if update.find(':') >= 0: - method, version = update.split(':') - else: - version = 'master' - self.run('git checkout %s' % version, cwd=location) - else: - # first remove the symlink needed when running from subversion - self.run('rm /usr/local/lib/python2.5/site-packages/boto') - self.run('easy_install %s' % update) - - def fetch_s3_file(self, s3_file): - try: - from boto.utils import fetch_file - f = fetch_file(s3_file) - path = os.path.join(self.working_dir, s3_file.split("/")[-1]) - open(path, "w").write(f.read()) - except: - boto.log.exception('Problem Retrieving file: %s' % s3_file) - path = None - return path - - def load_packages(self): - package_str = boto.config.get('Pyami', 'packages') - if package_str: - packages = package_str.split(',') - for package in packages: - package = package.strip() - if package.startswith('s3:'): - package = self.fetch_s3_file(package) - if package: - # if the "package" is really a .py file, it doesn't have to - # be installed, just being in the working dir is enough - if not package.endswith('.py'): - self.run('easy_install -Z %s' % package, exit_on_error=False) - - def main(self): - self.create_working_dir() - self.load_boto() - self.load_packages() - self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) - -if __name__ == "__main__": - # because bootstrap starts before any logging configuration can be loaded from - # the boto config files, we will manually enable logging to /var/log/boto.log - boto.set_file_logger('bootstrap', '/var/log/boto.log') - bs = Bootstrap() - bs.main() diff --git a/backup/src/boto/pyami/config.py b/backup/src/boto/pyami/config.py deleted file mode 100644 index f4613ab..0000000 --- a/backup/src/boto/pyami/config.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import StringIO, os, re -import ConfigParser -import boto - -BotoConfigPath = '/etc/boto.cfg' -BotoConfigLocations = [BotoConfigPath] -if 'BOTO_CONFIG' in os.environ: - BotoConfigLocations = [os.path.expanduser(os.environ['BOTO_CONFIG'])] -elif 'HOME' in os.environ: - UserConfigPath = os.path.expanduser('~/.boto') - BotoConfigLocations.append(UserConfigPath) -else: - UserConfigPath = None - -class Config(ConfigParser.SafeConfigParser): - - def __init__(self, path=None, fp=None, do_load=True): - ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', - 'debug' : '0'}) - if do_load: - if path: - self.load_from_path(path) - elif fp: - self.readfp(fp) - else: - self.read(BotoConfigLocations) - if "AWS_CREDENTIAL_FILE" in os.environ: - self.load_credential_file(os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])) - - def load_credential_file(self, path): - """Load a credential file as is setup like the Java utilities""" - c_data = StringIO.StringIO() - c_data.write("[Credentials]\n") - for line in open(path, "r").readlines(): - c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key")) - c_data.seek(0) - self.readfp(c_data) - - def load_from_path(self, path): - file = open(path) - for line in file.readlines(): - match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line) - if match: - extended_file = match.group(1) - (dir, file) = os.path.split(path) - self.load_from_path(os.path.join(dir, extended_file)) - self.read(path) - - def save_option(self, path, section, option, value): - """ - Write the specified Section.Option to the config file specified by path. - Replace any previous value. If the path doesn't exist, create it. - Also add the option the the in-memory config. - """ - config = ConfigParser.SafeConfigParser() - config.read(path) - if not config.has_section(section): - config.add_section(section) - config.set(section, option, value) - fp = open(path, 'w') - config.write(fp) - fp.close() - if not self.has_section(section): - self.add_section(section) - self.set(section, option, value) - - def save_user_option(self, section, option, value): - self.save_option(UserConfigPath, section, option, value) - - def save_system_option(self, section, option, value): - self.save_option(BotoConfigPath, section, option, value) - - def get_instance(self, name, default=None): - try: - val = self.get('Instance', name) - except: - val = default - return val - - def get_user(self, name, default=None): - try: - val = self.get('User', name) - except: - val = default - return val - - def getint_user(self, name, default=0): - try: - val = self.getint('User', name) - except: - val = default - return val - - def get_value(self, section, name, default=None): - return self.get(section, name, default) - - def get(self, section, name, default=None): - try: - val = ConfigParser.SafeConfigParser.get(self, section, name) - except: - val = default - return val - - def getint(self, section, name, default=0): - try: - val = ConfigParser.SafeConfigParser.getint(self, section, name) - except: - val = int(default) - return val - - def getfloat(self, section, name, default=0.0): - try: - val = ConfigParser.SafeConfigParser.getfloat(self, section, name) - except: - val = float(default) - return val - - def getbool(self, section, name, default=False): - if self.has_option(section, name): - val = self.get(section, name) - if val.lower() == 'true': - val = True - else: - val = False - else: - val = default - return val - - def setbool(self, section, name, value): - if value: - self.set(section, name, 'true') - else: - self.set(section, name, 'false') - - def dump(self): - s = StringIO.StringIO() - self.write(s) - print s.getvalue() - - def dump_safe(self, fp=None): - if not fp: - fp = StringIO.StringIO() - for section in self.sections(): - fp.write('[%s]\n' % section) - for option in self.options(section): - if option == 'aws_secret_access_key': - fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) - else: - fp.write('%s = %s\n' % (option, self.get(section, option))) - - def dump_to_sdb(self, domain_name, item_name): - import simplejson - sdb = boto.connect_sdb() - domain = sdb.lookup(domain_name) - if not domain: - domain = sdb.create_domain(domain_name) - item = domain.new_item(item_name) - item.active = False - for section in self.sections(): - d = {} - for option in self.options(section): - d[option] = self.get(section, option) - item[section] = simplejson.dumps(d) - item.save() - - def load_from_sdb(self, domain_name, item_name): - import simplejson - sdb = boto.connect_sdb() - domain = sdb.lookup(domain_name) - item = domain.get_item(item_name) - for section in item.keys(): - if not self.has_section(section): - self.add_section(section) - d = simplejson.loads(item[section]) - for attr_name in d.keys(): - attr_value = d[attr_name] - if attr_value == None: - attr_value = 'None' - if isinstance(attr_value, bool): - self.setbool(section, attr_name, attr_value) - else: - self.set(section, attr_name, attr_value) diff --git a/backup/src/boto/pyami/copybot.cfg b/backup/src/boto/pyami/copybot.cfg deleted file mode 100644 index cbfdc5a..0000000 --- a/backup/src/boto/pyami/copybot.cfg +++ /dev/null @@ -1,60 +0,0 @@ -# -# Your AWS Credentials -# -[Credentials] -aws_access_key_id = -aws_secret_access_key = - -# -# If you want to use a separate set of credentials when writing -# to the destination bucket, put them here -#dest_aws_access_key_id = -#dest_aws_secret_access_key = - -# -# Fill out this section if you want emails from CopyBot -# when it starts and stops -# -[Notification] -#smtp_host = -#smtp_user = -#smtp_pass = -#smtp_from = -#smtp_to = - -# -# If you leave this section as is, it will automatically -# update boto from subversion upon start up. -# If you don't want that to happen, comment this out -# -[Boto] -boto_location = /usr/local/boto -boto_update = svn:HEAD - -# -# This tells the Pyami code in boto what scripts -# to run during startup -# -[Pyami] -scripts = boto.pyami.copybot.CopyBot - -# -# Source bucket and Destination Bucket, obviously. -# If the Destination bucket does not exist, it will -# attempt to create it. -# If exit_on_completion is false, the instance -# will keep running after the copy operation is -# complete which might be handy for debugging. -# If copy_acls is false, the ACL's will not be -# copied with the objects to the new bucket. -# If replace_dst is false, copybot will not -# will only store the source file in the dest if -# that file does not already exist. If it's true -# it will replace it even if it does exist. -# -[CopyBot] -src_bucket = -dst_bucket = -exit_on_completion = true -copy_acls = true -replace_dst = true diff --git a/backup/src/boto/pyami/copybot.py b/backup/src/boto/pyami/copybot.py deleted file mode 100644 index ed397cb..0000000 --- a/backup/src/boto/pyami/copybot.py +++ /dev/null @@ -1,97 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import boto -from boto.pyami.scriptbase import ScriptBase -import os, StringIO - -class CopyBot(ScriptBase): - - def __init__(self): - ScriptBase.__init__(self) - self.wdir = boto.config.get('Pyami', 'working_dir') - self.log_file = '%s.log' % self.instance_id - self.log_path = os.path.join(self.wdir, self.log_file) - boto.set_file_logger(self.name, self.log_path) - self.src_name = boto.config.get(self.name, 'src_bucket') - self.dst_name = boto.config.get(self.name, 'dst_bucket') - self.replace = boto.config.getbool(self.name, 'replace_dst', True) - s3 = boto.connect_s3() - self.src = s3.lookup(self.src_name) - if not self.src: - boto.log.error('Source bucket does not exist: %s' % self.src_name) - dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) - if dest_access_key: - dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) - s3 = boto.connect(dest_access_key, dest_secret_key) - self.dst = s3.lookup(self.dst_name) - if not self.dst: - self.dst = s3.create_bucket(self.dst_name) - - def copy_bucket_acl(self): - if boto.config.get(self.name, 'copy_acls', True): - acl = self.src.get_xml_acl() - self.dst.set_xml_acl(acl) - - def copy_key_acl(self, src, dst): - if boto.config.get(self.name, 'copy_acls', True): - acl = src.get_xml_acl() - dst.set_xml_acl(acl) - - def copy_keys(self): - boto.log.info('src=%s' % self.src.name) - boto.log.info('dst=%s' % self.dst.name) - try: - for key in self.src: - if not self.replace: - exists = self.dst.lookup(key.name) - if exists: - boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name)) - continue - boto.log.info('copying %d bytes from key=%s' % (key.size, key.name)) - prefix, base = os.path.split(key.name) - path = os.path.join(self.wdir, base) - key.get_contents_to_filename(path) - new_key = self.dst.new_key(key.name) - new_key.set_contents_from_filename(path) - self.copy_key_acl(key, new_key) - os.unlink(path) - except: - boto.log.exception('Error copying key: %s' % key.name) - - def copy_log(self): - key = self.dst.new_key(self.log_file) - key.set_contents_from_filename(self.log_path) - - def main(self): - fp = StringIO.StringIO() - boto.config.dump_safe(fp) - self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue()) - if self.src and self.dst: - self.copy_keys() - if self.dst: - self.copy_log() - self.notify('%s (%s) Stopping' % (self.name, self.instance_id), - 'Copy Operation Complete') - if boto.config.getbool(self.name, 'exit_on_completion', True): - ec2 = boto.connect_ec2() - ec2.terminate_instances([self.instance_id]) - diff --git a/backup/src/boto/pyami/helloworld.py b/backup/src/boto/pyami/helloworld.py deleted file mode 100644 index 680873c..0000000 --- a/backup/src/boto/pyami/helloworld.py +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -from boto.pyami.scriptbase import ScriptBase - -class HelloWorld(ScriptBase): - - def main(self): - self.log('Hello World!!!') - diff --git a/backup/src/boto/pyami/installers/__init__.py b/backup/src/boto/pyami/installers/__init__.py deleted file mode 100644 index cc68926..0000000 --- a/backup/src/boto/pyami/installers/__init__.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -from boto.pyami.scriptbase import ScriptBase - - -class Installer(ScriptBase): - """ - Abstract base class for installers - """ - - def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None): - """ - Add an entry to the system crontab. - """ - raise NotImplementedError - - def add_init_script(self, file): - """ - Add this file to the init.d directory - """ - - def add_env(self, key, value): - """ - Add an environemnt variable - """ - raise NotImplementedError - - def stop(self, service_name): - """ - Stop a service. - """ - raise NotImplementedError - - def start(self, service_name): - """ - Start a service. - """ - raise NotImplementedError - - def install(self): - """ - Do whatever is necessary to "install" the package. - """ - raise NotImplementedError - diff --git a/backup/src/boto/pyami/installers/ubuntu/__init__.py b/backup/src/boto/pyami/installers/ubuntu/__init__.py deleted file mode 100644 index 60ee658..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/backup/src/boto/pyami/installers/ubuntu/apache.py b/backup/src/boto/pyami/installers/ubuntu/apache.py deleted file mode 100644 index febc2df..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/apache.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (c) 2008 Chris Moyer http://coredumped.org -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -from boto.pyami.installers.ubuntu.installer import Installer - -class Apache(Installer): - """ - Install apache2, mod_python, and libapache2-svn - """ - - def install(self): - self.run("apt-get update") - self.run('apt-get -y install apache2', notify=True, exit_on_error=True) - self.run('apt-get -y install libapache2-mod-python', notify=True, exit_on_error=True) - self.run('a2enmod rewrite', notify=True, exit_on_error=True) - self.run('a2enmod ssl', notify=True, exit_on_error=True) - self.run('a2enmod proxy', notify=True, exit_on_error=True) - self.run('a2enmod proxy_ajp', notify=True, exit_on_error=True) - - # Hard reboot the apache2 server to enable these module - self.stop("apache2") - self.start("apache2") - - def main(self): - self.install() diff --git a/backup/src/boto/pyami/installers/ubuntu/ebs.py b/backup/src/boto/pyami/installers/ubuntu/ebs.py deleted file mode 100644 index 204c9b1..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/ebs.py +++ /dev/null @@ -1,220 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -""" -Automated installer to attach, format and mount an EBS volume. -This installer assumes that you want the volume formatted as -an XFS file system. To drive this installer, you need the -following section in the boto config passed to the new instance. -You also need to install dateutil by listing python-dateutil -in the list of packages to be installed in the Pyami seciont -of your boto config file. - -If there is already a device mounted at the specified mount point, -the installer assumes that it is the ephemeral drive and unmounts -it, remounts it as /tmp and chmods it to 777. - -Config file section:: - - [EBS] - volume_id = - logical_volume_name = - device = - mount_point = - -""" -import boto -from boto.manage.volume import Volume -import os, time -from boto.pyami.installers.ubuntu.installer import Installer -from string import Template - -BackupScriptTemplate = """#!/usr/bin/env python -# Backup EBS volume -import boto -from boto.pyami.scriptbase import ScriptBase -import traceback - -class Backup(ScriptBase): - - def main(self): - try: - ec2 = boto.connect_ec2() - self.run("/usr/sbin/xfs_freeze -f ${mount_point}") - snapshot = ec2.create_snapshot('${volume_id}') - boto.log.info("Snapshot created: %s " % snapshot) - except Exception, e: - self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) - boto.log.info("Snapshot created: ${volume_id}") - except Exception, e: - self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) - finally: - self.run("/usr/sbin/xfs_freeze -u ${mount_point}") - -if __name__ == "__main__": - b = Backup() - b.main() -""" - -BackupCleanupScript= """#!/usr/bin/env python -import boto -from boto.manage.volume import Volume - -# Cleans Backups of EBS volumes - -for v in Volume.all(): - v.trim_snapshots(True) -""" - -class EBSInstaller(Installer): - """ - Set up the EBS stuff - """ - - def __init__(self, config_file=None): - Installer.__init__(self, config_file) - self.instance_id = boto.config.get('Instance', 'instance-id') - self.device = boto.config.get('EBS', 'device', '/dev/sdp') - self.volume_id = boto.config.get('EBS', 'volume_id') - self.logical_volume_name = boto.config.get('EBS', 'logical_volume_name') - self.mount_point = boto.config.get('EBS', 'mount_point', '/ebs') - - def attach(self): - ec2 = boto.connect_ec2() - if self.logical_volume_name: - # if a logical volume was specified, override the specified volume_id - # (if there was one) with the current AWS volume for the logical volume: - logical_volume = Volume.find(name = self.logical_volume_name).next() - self.volume_id = logical_volume._volume_id - volume = ec2.get_all_volumes([self.volume_id])[0] - # wait for the volume to be available. The volume may still be being created - # from a snapshot. - while volume.update() != 'available': - boto.log.info('Volume %s not yet available. Current status = %s.' % (volume.id, volume.status)) - time.sleep(5) - instance = ec2.get_all_instances([self.instance_id])[0].instances[0] - attempt_attach = True - while attempt_attach: - try: - ec2.attach_volume(self.volume_id, self.instance_id, self.device) - attempt_attach = False - except EC2ResponseError, e: - if e.error_code != 'IncorrectState': - # if there's an EC2ResonseError with the code set to IncorrectState, delay a bit for ec2 - # to realize the instance is running, then try again. Otherwise, raise the error: - boto.log.info('Attempt to attach the EBS volume %s to this instance (%s) returned %s. Trying again in a bit.' % (self.volume_id, self.instance_id, e.errors)) - time.sleep(2) - else: - raise e - boto.log.info('Attached volume %s to instance %s as device %s' % (self.volume_id, self.instance_id, self.device)) - # now wait for the volume device to appear - while not os.path.exists(self.device): - boto.log.info('%s still does not exist, waiting 2 seconds' % self.device) - time.sleep(2) - - def make_fs(self): - boto.log.info('make_fs...') - has_fs = self.run('fsck %s' % self.device) - if has_fs != 0: - self.run('mkfs -t xfs %s' % self.device) - - def create_backup_script(self): - t = Template(BackupScriptTemplate) - s = t.substitute(volume_id=self.volume_id, instance_id=self.instance_id, - mount_point=self.mount_point) - fp = open('/usr/local/bin/ebs_backup', 'w') - fp.write(s) - fp.close() - self.run('chmod +x /usr/local/bin/ebs_backup') - - def create_backup_cleanup_script(self): - fp = open('/usr/local/bin/ebs_backup_cleanup', 'w') - fp.write(BackupCleanupScript) - fp.close() - self.run('chmod +x /usr/local/bin/ebs_backup_cleanup') - - def handle_mount_point(self): - boto.log.info('handle_mount_point') - if not os.path.isdir(self.mount_point): - boto.log.info('making directory') - # mount directory doesn't exist so create it - self.run("mkdir %s" % self.mount_point) - else: - boto.log.info('directory exists already') - self.run('mount -l') - lines = self.last_command.output.split('\n') - for line in lines: - t = line.split() - if t and t[2] == self.mount_point: - # something is already mounted at the mount point - # unmount that and mount it as /tmp - if t[0] != self.device: - self.run('umount %s' % self.mount_point) - self.run('mount %s /tmp' % t[0]) - break - self.run('chmod 777 /tmp') - # Mount up our new EBS volume onto mount_point - self.run("mount %s %s" % (self.device, self.mount_point)) - self.run('xfs_growfs %s' % self.mount_point) - - def update_fstab(self): - f = open("/etc/fstab", "a") - f.write('%s\t%s\txfs\tdefaults 0 0\n' % (self.device, self.mount_point)) - f.close() - - def install(self): - # First, find and attach the volume - self.attach() - - # Install the xfs tools - self.run('apt-get -y install xfsprogs xfsdump') - - # Check to see if the filesystem was created or not - self.make_fs() - - # create the /ebs directory for mounting - self.handle_mount_point() - - # create the backup script - self.create_backup_script() - - # Set up the backup script - minute = boto.config.get('EBS', 'backup_cron_minute', '0') - hour = boto.config.get('EBS', 'backup_cron_hour', '4,16') - self.add_cron("ebs_backup", "/usr/local/bin/ebs_backup", minute=minute, hour=hour) - - # Set up the backup cleanup script - minute = boto.config.get('EBS', 'backup_cleanup_cron_minute') - hour = boto.config.get('EBS', 'backup_cleanup_cron_hour') - if (minute != None) and (hour != None): - self.create_backup_cleanup_script(); - self.add_cron("ebs_backup_cleanup", "/usr/local/bin/ebs_backup_cleanup", minute=minute, hour=hour) - - # Set up the fstab - self.update_fstab() - - def main(self): - if not os.path.exists(self.device): - self.install() - else: - boto.log.info("Device %s is already attached, skipping EBS Installer" % self.device) diff --git a/backup/src/boto/pyami/installers/ubuntu/installer.py b/backup/src/boto/pyami/installers/ubuntu/installer.py deleted file mode 100644 index 370d63f..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/installer.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import boto.pyami.installers -import os -import os.path -import stat -import boto -import random -from pwd import getpwnam - -class Installer(boto.pyami.installers.Installer): - """ - Base Installer class for Ubuntu-based AMI's - """ - def add_cron(self, name, command, minute="*", hour="*", mday="*", month="*", wday="*", who="root", env=None): - """ - Write a file to /etc/cron.d to schedule a command - env is a dict containing environment variables you want to set in the file - name will be used as the name of the file - """ - if minute == 'random': - minute = str(random.randrange(60)) - if hour == 'random': - hour = str(random.randrange(24)) - fp = open('/etc/cron.d/%s' % name, "w") - if env: - for key, value in env.items(): - fp.write('%s=%s\n' % (key, value)) - fp.write('%s %s %s %s %s %s %s\n' % (minute, hour, mday, month, wday, who, command)) - fp.close() - - def add_init_script(self, file, name): - """ - Add this file to the init.d directory - """ - f_path = os.path.join("/etc/init.d", name) - f = open(f_path, "w") - f.write(file) - f.close() - os.chmod(f_path, stat.S_IREAD| stat.S_IWRITE | stat.S_IEXEC) - self.run("/usr/sbin/update-rc.d %s defaults" % name) - - def add_env(self, key, value): - """ - Add an environemnt variable - For Ubuntu, the best place is /etc/environment. Values placed here do - not need to be exported. - """ - boto.log.info('Adding env variable: %s=%s' % (key, value)) - if not os.path.exists("/etc/environment.orig"): - self.run('cp /etc/environment /etc/environment.orig', notify=False, exit_on_error=False) - fp = open('/etc/environment', 'a') - fp.write('\n%s="%s"' % (key, value)) - fp.close() - os.environ[key] = value - - def stop(self, service_name): - self.run('/etc/init.d/%s stop' % service_name) - - def start(self, service_name): - self.run('/etc/init.d/%s start' % service_name) - - def create_user(self, user): - """ - Create a user on the local system - """ - self.run("useradd -m %s" % user) - usr = getpwnam(user) - return usr - - - def install(self): - """ - This is the only method you need to override - """ - raise NotImplementedError - diff --git a/backup/src/boto/pyami/installers/ubuntu/mysql.py b/backup/src/boto/pyami/installers/ubuntu/mysql.py deleted file mode 100644 index 490e5db..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/mysql.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -""" -This installer will install mysql-server on an Ubuntu machine. -In addition to the normal installation done by apt-get, it will -also configure the new MySQL server to store it's data files in -a different location. By default, this is /mnt but that can be -configured in the [MySQL] section of the boto config file passed -to the instance. -""" -from boto.pyami.installers.ubuntu.installer import Installer -import os -import boto -from boto.utils import ShellCommand -from ConfigParser import SafeConfigParser -import time - -ConfigSection = """ -[MySQL] -root_password = -data_dir = -""" - -class MySQL(Installer): - - def install(self): - self.run('apt-get update') - self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True) - -# def set_root_password(self, password=None): -# if not password: -# password = boto.config.get('MySQL', 'root_password') -# if password: -# self.run('mysqladmin -u root password %s' % password) -# return password - - def change_data_dir(self, password=None): - data_dir = boto.config.get('MySQL', 'data_dir', '/mnt') - fresh_install = False; - is_mysql_running_command = ShellCommand('mysqladmin ping') # exit status 0 if mysql is running - is_mysql_running_command.run() - if is_mysql_running_command.getStatus() == 0: - # mysql is running. This is the state apt-get will leave it in. If it isn't running, - # that means mysql was already installed on the AMI and there's no need to stop it, - # saving 40 seconds on instance startup. - time.sleep(10) #trying to stop mysql immediately after installing it fails - # We need to wait until mysql creates the root account before we kill it - # or bad things will happen - i = 0 - while self.run("echo 'quit' | mysql -u root") != 0 and i<5: - time.sleep(5) - i = i + 1 - self.run('/etc/init.d/mysql stop') - self.run("pkill -9 mysql") - - mysql_path = os.path.join(data_dir, 'mysql') - if not os.path.exists(mysql_path): - self.run('mkdir %s' % mysql_path) - fresh_install = True; - self.run('chown -R mysql:mysql %s' % mysql_path) - fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w') - fp.write('# created by pyami\n') - fp.write('# use the %s volume for data\n' % data_dir) - fp.write('[mysqld]\n') - fp.write('datadir = %s\n' % mysql_path) - fp.write('log_bin = %s\n' % os.path.join(mysql_path, 'mysql-bin.log')) - fp.close() - if fresh_install: - self.run('cp -pr /var/lib/mysql/* %s/' % mysql_path) - self.start('mysql') - else: - #get the password ubuntu expects to use: - config_parser = SafeConfigParser() - config_parser.read('/etc/mysql/debian.cnf') - password = config_parser.get('client', 'password') - # start the mysql deamon, then mysql with the required grant statement piped into it: - self.start('mysql') - time.sleep(10) #time for mysql to start - grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password - while self.run(grant_command) != 0: - time.sleep(5) - # leave mysqld running - - def main(self): - self.install() - # change_data_dir runs 'mysql -u root' which assumes there is no mysql password, i - # and changing that is too ugly to be worth it: - #self.set_root_password() - self.change_data_dir() - diff --git a/backup/src/boto/pyami/installers/ubuntu/trac.py b/backup/src/boto/pyami/installers/ubuntu/trac.py deleted file mode 100644 index ef83af7..0000000 --- a/backup/src/boto/pyami/installers/ubuntu/trac.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2008 Chris Moyer http://coredumped.org -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -from boto.pyami.installers.ubuntu.installer import Installer -import boto -import os - -class Trac(Installer): - """ - Install Trac and DAV-SVN - Sets up a Vhost pointing to [Trac]->home - Using the config parameter [Trac]->hostname - Sets up a trac environment for every directory found under [Trac]->data_dir - - [Trac] - name = My Foo Server - hostname = trac.foo.com - home = /mnt/sites/trac - data_dir = /mnt/trac - svn_dir = /mnt/subversion - server_admin = root@foo.com - sdb_auth_domain = users - # Optional - SSLCertificateFile = /mnt/ssl/foo.crt - SSLCertificateKeyFile = /mnt/ssl/foo.key - SSLCertificateChainFile = /mnt/ssl/FooCA.crt - - """ - - def install(self): - self.run('apt-get -y install trac', notify=True, exit_on_error=True) - self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) - self.run("a2enmod ssl") - self.run("a2enmod mod_python") - self.run("a2enmod dav_svn") - self.run("a2enmod rewrite") - # Make sure that boto.log is writable by everyone so that subversion post-commit hooks can - # write to it. - self.run("touch /var/log/boto.log") - self.run("chmod a+w /var/log/boto.log") - - def setup_vhost(self): - domain = boto.config.get("Trac", "hostname").strip() - if domain: - domain_info = domain.split('.') - cnf = open("/etc/apache2/sites-available/%s" % domain_info[0], "w") - cnf.write("NameVirtualHost *:80\n") - if boto.config.get("Trac", "SSLCertificateFile"): - cnf.write("NameVirtualHost *:443\n\n") - cnf.write("\n") - cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) - cnf.write("\tServerName %s\n" % domain) - cnf.write("\tRewriteEngine On\n") - cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) - cnf.write("\n\n") - - cnf.write("\n") - else: - cnf.write("\n") - - cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) - cnf.write("\tServerName %s\n" % domain) - cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) - - cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) - cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") - cnf.write("\t\tAllowOverride All\n") - cnf.write("\t\tOrder allow,deny\n") - cnf.write("\t\tallow from all\n") - cnf.write("\t\n") - - cnf.write("\t\n") - cnf.write("\t\tAuthType Basic\n") - cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) - cnf.write("\t\tRequire valid-user\n") - cnf.write("\t\tAuthUserFile /mnt/apache/passwd/passwords\n") - cnf.write("\t\n") - - data_dir = boto.config.get("Trac", "data_dir") - for env in os.listdir(data_dir): - if(env[0] != "."): - cnf.write("\t\n" % env) - cnf.write("\t\tSetHandler mod_python\n") - cnf.write("\t\tPythonInterpreter main_interpreter\n") - cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") - cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) - cnf.write("\t\tPythonOption TracUriRoot /trac/%s\n" % env) - cnf.write("\t\n") - - svn_dir = boto.config.get("Trac", "svn_dir") - for env in os.listdir(svn_dir): - if(env[0] != "."): - cnf.write("\t\n" % env) - cnf.write("\t\tDAV svn\n") - cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) - cnf.write("\t\n") - - cnf.write("\tErrorLog /var/log/apache2/error.log\n") - cnf.write("\tLogLevel warn\n") - cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") - cnf.write("\tServerSignature On\n") - SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") - if SSLCertificateFile: - cnf.write("\tSSLEngine On\n") - cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) - - SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") - if SSLCertificateKeyFile: - cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) - - SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") - if SSLCertificateChainFile: - cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) - cnf.write("\n") - cnf.close() - self.run("a2ensite %s" % domain_info[0]) - self.run("/etc/init.d/apache2 force-reload") - - def main(self): - self.install() - self.setup_vhost() diff --git a/backup/src/boto/pyami/launch_ami.py b/backup/src/boto/pyami/launch_ami.py deleted file mode 100644 index 243d56d..0000000 --- a/backup/src/boto/pyami/launch_ami.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import getopt -import sys -import imp -import time -import boto - -usage_string = """ -SYNOPSIS - launch_ami.py -a ami_id [-b script_bucket] [-s script_name] - [-m module] [-c class_name] [-r] - [-g group] [-k key_name] [-n num_instances] - [-w] [extra_data] - Where: - ami_id - the id of the AMI you wish to launch - module - The name of the Python module containing the class you - want to run when the instance is started. If you use this - option the Python module must already be stored on the - instance in a location that is on the Python path. - script_file - The name of a local Python module that you would like - to have copied to S3 and then run on the instance - when it is started. The specified module must be - import'able (i.e. in your local Python path). It - will then be copied to the specified bucket in S3 - (see the -b option). Once the new instance(s) - start up the script will be copied from S3 and then - run locally on the instance. - class_name - The name of the class to be instantiated within the - module or script file specified. - script_bucket - the name of the bucket in which the script will be - stored - group - the name of the security group the instance will run in - key_name - the name of the keypair to use when launching the AMI - num_instances - how many instances of the AMI to launch (default 1) - input_queue_name - Name of SQS to read input messages from - output_queue_name - Name of SQS to write output messages to - extra_data - additional name-value pairs that will be passed as - userdata to the newly launched instance. These should - be of the form "name=value" - The -r option reloads the Python module to S3 without launching - another instance. This can be useful during debugging to allow - you to test a new version of your script without shutting down - your instance and starting up another one. - The -w option tells the script to run synchronously, meaning to - wait until the instance is actually up and running. It then prints - the IP address and internal and external DNS names before exiting. -""" - -def usage(): - print usage_string - sys.exit() - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', - ['ami', 'bucket', 'class', 'group', 'help', - 'inputqueue', 'keypair', 'module', - 'numinstances', 'outputqueue', - 'reload', 'script_name', 'wait']) - except: - usage() - params = {'module_name' : None, - 'script_name' : None, - 'class_name' : None, - 'script_bucket' : None, - 'group' : 'default', - 'keypair' : None, - 'ami' : None, - 'num_instances' : 1, - 'input_queue_name' : None, - 'output_queue_name' : None} - reload = None - wait = None - for o, a in opts: - if o in ('-a', '--ami'): - params['ami'] = a - if o in ('-b', '--bucket'): - params['script_bucket'] = a - if o in ('-c', '--class'): - params['class_name'] = a - if o in ('-g', '--group'): - params['group'] = a - if o in ('-h', '--help'): - usage() - if o in ('-i', '--inputqueue'): - params['input_queue_name'] = a - if o in ('-k', '--keypair'): - params['keypair'] = a - if o in ('-m', '--module'): - params['module_name'] = a - if o in ('-n', '--num_instances'): - params['num_instances'] = int(a) - if o in ('-o', '--outputqueue'): - params['output_queue_name'] = a - if o in ('-r', '--reload'): - reload = True - if o in ('-s', '--script'): - params['script_name'] = a - if o in ('-w', '--wait'): - wait = True - - # check required fields - required = ['ami'] - for pname in required: - if not params.get(pname, None): - print '%s is required' % pname - usage() - if params['script_name']: - # first copy the desired module file to S3 bucket - if reload: - print 'Reloading module %s to S3' % params['script_name'] - else: - print 'Copying module %s to S3' % params['script_name'] - l = imp.find_module(params['script_name']) - c = boto.connect_s3() - bucket = c.get_bucket(params['script_bucket']) - key = bucket.new_key(params['script_name']+'.py') - key.set_contents_from_file(l[0]) - params['script_md5'] = key.md5 - # we have everything we need, now build userdata string - l = [] - for k, v in params.items(): - if v: - l.append('%s=%s' % (k, v)) - c = boto.connect_ec2() - l.append('aws_access_key_id=%s' % c.aws_access_key_id) - l.append('aws_secret_access_key=%s' % c.aws_secret_access_key) - for kv in args: - l.append(kv) - s = '|'.join(l) - if not reload: - rs = c.get_all_images([params['ami']]) - img = rs[0] - r = img.run(user_data=s, key_name=params['keypair'], - security_groups=[params['group']], - max_count=params.get('num_instances', 1)) - print 'AMI: %s - %s (Started)' % (params['ami'], img.location) - print 'Reservation %s contains the following instances:' % r.id - for i in r.instances: - print '\t%s' % i.id - if wait: - running = False - while not running: - time.sleep(30) - [i.update() for i in r.instances] - status = [i.state for i in r.instances] - print status - if status.count('running') == len(r.instances): - running = True - for i in r.instances: - print 'Instance: %s' % i.ami_launch_index - print 'Public DNS Name: %s' % i.public_dns_name - print 'Private DNS Name: %s' % i.private_dns_name - -if __name__ == "__main__": - main() - diff --git a/backup/src/boto/pyami/scriptbase.py b/backup/src/boto/pyami/scriptbase.py deleted file mode 100644 index 90522ca..0000000 --- a/backup/src/boto/pyami/scriptbase.py +++ /dev/null @@ -1,44 +0,0 @@ -import os -import sys -from boto.utils import ShellCommand, get_ts -import boto -import boto.utils - -class ScriptBase: - - def __init__(self, config_file=None): - self.instance_id = boto.config.get('Instance', 'instance-id', 'default') - self.name = self.__class__.__name__ - self.ts = get_ts() - if config_file: - boto.config.read(config_file) - - def notify(self, subject, body=''): - boto.utils.notify(subject, body) - - def mkdir(self, path): - if not os.path.isdir(path): - try: - os.mkdir(path) - except: - boto.log.error('Error creating directory: %s' % path) - - def umount(self, path): - if os.path.ismount(path): - self.run('umount %s' % path) - - def run(self, command, notify=True, exit_on_error=False, cwd=None): - self.last_command = ShellCommand(command, cwd=cwd) - if self.last_command.status != 0: - boto.log.error('Error running command: "%s". Output: "%s"' % (command, self.last_command.output)) - if notify: - self.notify('Error encountered', \ - 'Error running the following command:\n\t%s\n\nCommand output:\n\t%s' % \ - (command, self.last_command.output)) - if exit_on_error: - sys.exit(-1) - return self.last_command.status - - def main(self): - pass - diff --git a/backup/src/boto/pyami/startup.py b/backup/src/boto/pyami/startup.py deleted file mode 100644 index 2093151..0000000 --- a/backup/src/boto/pyami/startup.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# -import sys -import boto -from boto.utils import find_class -from boto import config -from boto.pyami.scriptbase import ScriptBase - - -class Startup(ScriptBase): - - def run_scripts(self): - scripts = config.get('Pyami', 'scripts') - if scripts: - for script in scripts.split(','): - script = script.strip(" ") - try: - pos = script.rfind('.') - if pos > 0: - mod_name = script[0:pos] - cls_name = script[pos+1:] - cls = find_class(mod_name, cls_name) - boto.log.info('Running Script: %s' % script) - s = cls() - s.main() - else: - boto.log.warning('Trouble parsing script: %s' % script) - except Exception, e: - boto.log.exception('Problem Running Script: %s. Startup process halting.' % script) - raise e - - def main(self): - self.run_scripts() - self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) - -if __name__ == "__main__": - if not config.has_section('loggers'): - boto.set_file_logger('startup', '/var/log/boto.log') - sys.path.append(config.get('Pyami', 'working_dir')) - su = Startup() - su.main() diff --git a/backup/src/boto/rds/__init__.py b/backup/src/boto/rds/__init__.py deleted file mode 100644 index 940815d..0000000 --- a/backup/src/boto/rds/__init__.py +++ /dev/null @@ -1,972 +0,0 @@ -# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import boto.utils -import urllib -from boto.connection import AWSQueryConnection -from boto.rds.dbinstance import DBInstance -from boto.rds.dbsecuritygroup import DBSecurityGroup -from boto.rds.parametergroup import ParameterGroup -from boto.rds.dbsnapshot import DBSnapshot -from boto.rds.event import Event -from boto.rds.regioninfo import RDSRegionInfo - -def regions(): - """ - Get all available regions for the RDS service. - - :rtype: list - :return: A list of :class:`boto.rds.regioninfo.RDSRegionInfo` - """ - return [RDSRegionInfo(name='us-east-1', - endpoint='rds.amazonaws.com'), - RDSRegionInfo(name='eu-west-1', - endpoint='eu-west-1.rds.amazonaws.com'), - RDSRegionInfo(name='us-west-1', - endpoint='us-west-1.rds.amazonaws.com'), - RDSRegionInfo(name='ap-southeast-1', - endpoint='ap-southeast-1.rds.amazonaws.com') - ] - -def connect_to_region(region_name): - for region in regions(): - if region.name == region_name: - return region.connect() - return None - -#boto.set_stream_logger('rds') - -class RDSConnection(AWSQueryConnection): - - DefaultRegionName = 'us-east-1' - DefaultRegionEndpoint = 'rds.amazonaws.com' - APIVersion = '2009-10-16' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - if not region: - region = RDSRegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, - proxy_pass, self.region.endpoint, debug, - https_connection_factory, path) - - def _required_auth_capability(self): - return ['rds'] - - # DB Instance methods - - def get_all_dbinstances(self, instance_id=None, max_records=None, - marker=None): - """ - Retrieve all the DBInstances in your account. - - :type instance_id: str - :param instance_id: DB Instance identifier. If supplied, only information - this instance will be returned. Otherwise, info - about all DB Instances will be returned. - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: list - :return: A list of :class:`boto.rds.dbinstance.DBInstance` - """ - params = {} - if instance_id: - params['DBInstanceIdentifier'] = instance_id - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - return self.get_list('DescribeDBInstances', params, [('DBInstance', DBInstance)]) - - def create_dbinstance(self, id, allocated_storage, instance_class, - master_username, master_password, port=3306, - engine='MySQL5.1', db_name=None, param_group=None, - security_groups=None, availability_zone=None, - preferred_maintenance_window=None, - backup_retention_period=None, - preferred_backup_window=None, - multi_az=False, - engine_version=None, - auto_minor_version_upgrade=True): - """ - Create a new DBInstance. - - :type id: str - :param id: Unique identifier for the new instance. - Must contain 1-63 alphanumeric characters. - First character must be a letter. - May not end with a hyphen or contain two consecutive hyphens - - :type allocated_storage: int - :param allocated_storage: Initially allocated storage size, in GBs. - Valid values are [5-1024] - - :type instance_class: str - :param instance_class: The compute and memory capacity of the DBInstance. - - Valid values are: - - * db.m1.small - * db.m1.large - * db.m1.xlarge - * db.m2.xlarge - * db.m2.2xlarge - * db.m2.4xlarge - - :type engine: str - :param engine: Name of database engine. Must be MySQL5.1 for now. - - :type master_username: str - :param master_username: Name of master user for the DBInstance. - Must be 1-15 alphanumeric characters, first - must be a letter. - - :type master_password: str - :param master_password: Password of master user for the DBInstance. - Must be 4-16 alphanumeric characters. - - :type port: int - :param port: Port number on which database accepts connections. - Valid values [1115-65535]. Defaults to 3306. - - :type db_name: str - :param db_name: Name of a database to create when the DBInstance - is created. Default is to create no databases. - - :type param_group: str - :param param_group: Name of DBParameterGroup to associate with - this DBInstance. If no groups are specified - no parameter groups will be used. - - :type security_groups: list of str or list of DBSecurityGroup objects - :param security_groups: List of names of DBSecurityGroup to authorize on - this DBInstance. - - :type availability_zone: str - :param availability_zone: Name of the availability zone to place - DBInstance into. - - :type preferred_maintenance_window: str - :param preferred_maintenance_window: The weekly time range (in UTC) - during which maintenance can occur. - Default is Sun:05:00-Sun:09:00 - - :type backup_retention_period: int - :param backup_retention_period: The number of days for which automated - backups are retained. Setting this to - zero disables automated backups. - - :type preferred_backup_window: str - :param preferred_backup_window: The daily time range during which - automated backups are created (if - enabled). Must be in h24:mi-hh24:mi - format (UTC). - - :type multi_az: bool - :param multi_az: If True, specifies the DB Instance will be - deployed in multiple availability zones. - - :type engine_version: str - :param engine_version: Version number of the database engine to use. - - :type auto_minor_version_upgrade: bool - :param auto_minor_version_upgrade: Indicates that minor engine - upgrades will be applied - automatically to the Read Replica - during the maintenance window. - Default is True. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The new db instance. - """ - params = {'DBInstanceIdentifier' : id, - 'AllocatedStorage' : allocated_storage, - 'DBInstanceClass' : instance_class, - 'Engine' : engine, - 'MasterUsername' : master_username, - 'MasterUserPassword' : master_password} - if port: - params['Port'] = port - if db_name: - params['DBName'] = db_name - if param_group: - params['DBParameterGroupName'] = param_group - if security_groups: - l = [] - for group in security_groups: - if isinstance(group, DBSecurityGroup): - l.append(group.name) - else: - l.append(group) - self.build_list_params(params, l, 'DBSecurityGroups.member') - if availability_zone: - params['AvailabilityZone'] = availability_zone - if preferred_maintenance_window: - params['PreferredMaintenanceWindow'] = preferred_maintenance_window - if backup_retention_period: - params['BackupRetentionPeriod'] = backup_retention_period - if preferred_backup_window: - params['PreferredBackupWindow'] = preferred_backup_window - if multi_az: - params['MultiAZ'] = 'true' - if engine_version: - params['EngineVersion'] = engine_version - if auto_minor_version_upgrade is False: - params['AutoMinorVersionUpgrade'] = 'false' - - return self.get_object('CreateDBInstance', params, DBInstance) - - def create_dbinstance_read_replica(self, id, source_id, - instance_class=None, - port=3306, - availability_zone=None, - auto_minor_version_upgrade=None): - """ - Create a new DBInstance Read Replica. - - :type id: str - :param id: Unique identifier for the new instance. - Must contain 1-63 alphanumeric characters. - First character must be a letter. - May not end with a hyphen or contain two consecutive hyphens - - :type source_id: str - :param source_id: Unique identifier for the DB Instance for which this - DB Instance will act as a Read Replica. - - :type instance_class: str - :param instance_class: The compute and memory capacity of the - DBInstance. Default is to inherit from - the source DB Instance. - - Valid values are: - - * db.m1.small - * db.m1.large - * db.m1.xlarge - * db.m2.xlarge - * db.m2.2xlarge - * db.m2.4xlarge - - :type port: int - :param port: Port number on which database accepts connections. - Default is to inherit from source DB Instance. - Valid values [1115-65535]. Defaults to 3306. - - :type availability_zone: str - :param availability_zone: Name of the availability zone to place - DBInstance into. - - :type auto_minor_version_upgrade: bool - :param auto_minor_version_upgrade: Indicates that minor engine - upgrades will be applied - automatically to the Read Replica - during the maintenance window. - Default is to inherit this value - from the source DB Instance. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The new db instance. - """ - params = {'DBInstanceIdentifier' : id, - 'SourceDBInstanceIdentifier' : source_id} - if instance_class: - params['DBInstanceClass'] = instance_class - if port: - params['Port'] = port - if availability_zone: - params['AvailabilityZone'] = availability_zone - if auto_minor_version_upgrade is not None: - if auto_minor_version_upgrade is True: - params['AutoMinorVersionUpgrade'] = 'true' - else: - params['AutoMinorVersionUpgrade'] = 'false' - - return self.get_object('CreateDBInstanceReadReplica', - params, DBInstance) - - def modify_dbinstance(self, id, param_group=None, security_groups=None, - preferred_maintenance_window=None, - master_password=None, allocated_storage=None, - instance_class=None, - backup_retention_period=None, - preferred_backup_window=None, - multi_az=False, - apply_immediately=False): - """ - Modify an existing DBInstance. - - :type id: str - :param id: Unique identifier for the new instance. - - :type security_groups: list of str or list of DBSecurityGroup objects - :param security_groups: List of names of DBSecurityGroup to authorize on - this DBInstance. - - :type preferred_maintenance_window: str - :param preferred_maintenance_window: The weekly time range (in UTC) - during which maintenance can - occur. - Default is Sun:05:00-Sun:09:00 - - :type master_password: str - :param master_password: Password of master user for the DBInstance. - Must be 4-15 alphanumeric characters. - - :type allocated_storage: int - :param allocated_storage: The new allocated storage size, in GBs. - Valid values are [5-1024] - - :type instance_class: str - :param instance_class: The compute and memory capacity of the - DBInstance. Changes will be applied at - next maintenance window unless - apply_immediately is True. - - Valid values are: - - * db.m1.small - * db.m1.large - * db.m1.xlarge - * db.m2.xlarge - * db.m2.2xlarge - * db.m2.4xlarge - - :type apply_immediately: bool - :param apply_immediately: If true, the modifications will be applied - as soon as possible rather than waiting for - the next preferred maintenance window. - - :type backup_retention_period: int - :param backup_retention_period: The number of days for which automated - backups are retained. Setting this to - zero disables automated backups. - - :type preferred_backup_window: str - :param preferred_backup_window: The daily time range during which - automated backups are created (if - enabled). Must be in h24:mi-hh24:mi - format (UTC). - - :type multi_az: bool - :param multi_az: If True, specifies the DB Instance will be - deployed in multiple availability zones. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The modified db instance. - """ - params = {'DBInstanceIdentifier' : id} - if param_group: - params['DBParameterGroupName'] = param_group - if security_groups: - l = [] - for group in security_groups: - if isinstance(group, DBSecurityGroup): - l.append(group.name) - else: - l.append(group) - self.build_list_params(params, l, 'DBSecurityGroups.member') - if preferred_maintenance_window: - params['PreferredMaintenanceWindow'] = preferred_maintenance_window - if master_password: - params['MasterUserPassword'] = master_password - if allocated_storage: - params['AllocatedStorage'] = allocated_storage - if instance_class: - params['DBInstanceClass'] = instance_class - if backup_retention_period: - params['BackupRetentionPeriod'] = backup_retention_period - if preferred_backup_window: - params['PreferredBackupWindow'] = preferred_backup_window - if multi_az: - params['MultiAZ'] = 'true' - if apply_immediately: - params['ApplyImmediately'] = 'true' - - return self.get_object('ModifyDBInstance', params, DBInstance) - - def delete_dbinstance(self, id, skip_final_snapshot=False, - final_snapshot_id=''): - """ - Delete an existing DBInstance. - - :type id: str - :param id: Unique identifier for the new instance. - - :type skip_final_snapshot: bool - :param skip_final_snapshot: This parameter determines whether a final - db snapshot is created before the instance - is deleted. If True, no snapshot is created. - If False, a snapshot is created before - deleting the instance. - - :type final_snapshot_id: str - :param final_snapshot_id: If a final snapshot is requested, this - is the identifier used for that snapshot. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The deleted db instance. - """ - params = {'DBInstanceIdentifier' : id} - if skip_final_snapshot: - params['SkipFinalSnapshot'] = 'true' - else: - params['SkipFinalSnapshot'] = 'false' - params['FinalDBSnapshotIdentifier'] = final_snapshot_id - return self.get_object('DeleteDBInstance', params, DBInstance) - - - def reboot_dbinstance(self, id): - """ - Reboot DBInstance. - - :type id: str - :param id: Unique identifier of the instance. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The rebooting db instance. - """ - params = {'DBInstanceIdentifier' : id} - return self.get_object('RebootDBInstance', params, DBInstance) - - # DBParameterGroup methods - - def get_all_dbparameter_groups(self, groupname=None, max_records=None, - marker=None): - """ - Get all parameter groups associated with your account in a region. - - :type groupname: str - :param groupname: The name of the DBParameter group to retrieve. - If not provided, all DBParameter groups will be returned. - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: list - :return: A list of :class:`boto.ec2.parametergroup.ParameterGroup` - """ - params = {} - if groupname: - params['DBParameterGroupName'] = groupname - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - return self.get_list('DescribeDBParameterGroups', params, - [('DBParameterGroup', ParameterGroup)]) - - def get_all_dbparameters(self, groupname, source=None, - max_records=None, marker=None): - """ - Get all parameters associated with a ParameterGroup - - :type groupname: str - :param groupname: The name of the DBParameter group to retrieve. - - :type source: str - :param source: Specifies which parameters to return. - If not specified, all parameters will be returned. - Valid values are: user|system|engine-default - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: :class:`boto.ec2.parametergroup.ParameterGroup` - :return: The ParameterGroup - """ - params = {'DBParameterGroupName' : groupname} - if source: - params['Source'] = source - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - pg = self.get_object('DescribeDBParameters', params, ParameterGroup) - pg.name = groupname - return pg - - def create_parameter_group(self, name, engine='MySQL5.1', description=''): - """ - Create a new dbparameter group for your account. - - :type name: string - :param name: The name of the new dbparameter group - - :type engine: str - :param engine: Name of database engine. Must be MySQL5.1 for now. - - :type description: string - :param description: The description of the new security group - - :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` - :return: The newly created DBSecurityGroup - """ - params = {'DBParameterGroupName': name, - 'Engine': engine, - 'Description' : description} - return self.get_object('CreateDBParameterGroup', params, ParameterGroup) - - def modify_parameter_group(self, name, parameters=None): - """ - Modify a parameter group for your account. - - :type name: string - :param name: The name of the new parameter group - - :type parameters: list of :class:`boto.rds.parametergroup.Parameter` - :param parameters: The new parameters - - :rtype: :class:`boto.rds.parametergroup.ParameterGroup` - :return: The newly created ParameterGroup - """ - params = {'DBParameterGroupName': name} - for i in range(0, len(parameters)): - parameter = parameters[i] - parameter.merge(params, i+1) - return self.get_list('ModifyDBParameterGroup', params, ParameterGroup) - - def reset_parameter_group(self, name, reset_all_params=False, parameters=None): - """ - Resets some or all of the parameters of a ParameterGroup to the - default value - - :type key_name: string - :param key_name: The name of the ParameterGroup to reset - - :type parameters: list of :class:`boto.rds.parametergroup.Parameter` - :param parameters: The parameters to reset. If not supplied, all parameters - will be reset. - """ - params = {'DBParameterGroupName':name} - if reset_all_params: - params['ResetAllParameters'] = 'true' - else: - params['ResetAllParameters'] = 'false' - for i in range(0, len(parameters)): - parameter = parameters[i] - parameter.merge(params, i+1) - return self.get_status('ResetDBParameterGroup', params) - - def delete_parameter_group(self, name): - """ - Delete a DBSecurityGroup from your account. - - :type key_name: string - :param key_name: The name of the DBSecurityGroup to delete - """ - params = {'DBParameterGroupName':name} - return self.get_status('DeleteDBParameterGroup', params) - - # DBSecurityGroup methods - - def get_all_dbsecurity_groups(self, groupname=None, max_records=None, - marker=None): - """ - Get all security groups associated with your account in a region. - - :type groupnames: list - :param groupnames: A list of the names of security groups to retrieve. - If not provided, all security groups will be returned. - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: list - :return: A list of :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` - """ - params = {} - if groupname: - params['DBSecurityGroupName'] = groupname - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - return self.get_list('DescribeDBSecurityGroups', params, - [('DBSecurityGroup', DBSecurityGroup)]) - - def create_dbsecurity_group(self, name, description=None): - """ - Create a new security group for your account. - This will create the security group within the region you - are currently connected to. - - :type name: string - :param name: The name of the new security group - - :type description: string - :param description: The description of the new security group - - :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` - :return: The newly created DBSecurityGroup - """ - params = {'DBSecurityGroupName':name} - if description: - params['DBSecurityGroupDescription'] = description - group = self.get_object('CreateDBSecurityGroup', params, DBSecurityGroup) - group.name = name - group.description = description - return group - - def delete_dbsecurity_group(self, name): - """ - Delete a DBSecurityGroup from your account. - - :type key_name: string - :param key_name: The name of the DBSecurityGroup to delete - """ - params = {'DBSecurityGroupName':name} - return self.get_status('DeleteDBSecurityGroup', params) - - def authorize_dbsecurity_group(self, group_name, cidr_ip=None, - ec2_security_group_name=None, - ec2_security_group_owner_id=None): - """ - Add a new rule to an existing security group. - You need to pass in either src_security_group_name and - src_security_group_owner_id OR a CIDR block but not both. - - :type group_name: string - :param group_name: The name of the security group you are adding - the rule to. - - :type ec2_security_group_name: string - :param ec2_security_group_name: The name of the EC2 security group you are - granting access to. - - :type ec2_security_group_owner_id: string - :param ec2_security_group_owner_id: The ID of the owner of the EC2 security - group you are granting access to. - - :type cidr_ip: string - :param cidr_ip: The CIDR block you are providing access to. - See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - - :rtype: bool - :return: True if successful. - """ - params = {'DBSecurityGroupName':group_name} - if ec2_security_group_name: - params['EC2SecurityGroupName'] = ec2_security_group_name - if ec2_security_group_owner_id: - params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id - if cidr_ip: - params['CIDRIP'] = urllib.quote(cidr_ip) - return self.get_object('AuthorizeDBSecurityGroupIngress', params, DBSecurityGroup) - - def revoke_dbsecurity_group(self, group_name, ec2_security_group_name=None, - ec2_security_group_owner_id=None, cidr_ip=None): - """ - Remove an existing rule from an existing security group. - You need to pass in either ec2_security_group_name and - ec2_security_group_owner_id OR a CIDR block. - - :type group_name: string - :param group_name: The name of the security group you are removing - the rule from. - - :type ec2_security_group_name: string - :param ec2_security_group_name: The name of the EC2 security group from which - you are removing access. - - :type ec2_security_group_owner_id: string - :param ec2_security_group_owner_id: The ID of the owner of the EC2 security - from which you are removing access. - - :type cidr_ip: string - :param cidr_ip: The CIDR block from which you are removing access. - See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing - - :rtype: bool - :return: True if successful. - """ - params = {'DBSecurityGroupName':group_name} - if ec2_security_group_name: - params['EC2SecurityGroupName'] = ec2_security_group_name - if ec2_security_group_owner_id: - params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id - if cidr_ip: - params['CIDRIP'] = cidr_ip - return self.get_object('RevokeDBSecurityGroupIngress', params, DBSecurityGroup) - - # For backwards compatibility. This method was improperly named - # in previous versions. I have renamed it to match the others. - revoke_security_group = revoke_dbsecurity_group - - # DBSnapshot methods - - def get_all_dbsnapshots(self, snapshot_id=None, instance_id=None, - max_records=None, marker=None): - """ - Get information about DB Snapshots. - - :type snapshot_id: str - :param snapshot_id: The unique identifier of an RDS snapshot. - If not provided, all RDS snapshots will be returned. - - :type instance_id: str - :param instance_id: The identifier of a DBInstance. If provided, - only the DBSnapshots related to that instance will - be returned. - If not provided, all RDS snapshots will be returned. - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: list - :return: A list of :class:`boto.rds.dbsnapshot.DBSnapshot` - """ - params = {} - if snapshot_id: - params['DBSnapshotIdentifier'] = snapshot_id - if instance_id: - params['DBInstanceIdentifier'] = instance_id - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - return self.get_list('DescribeDBSnapshots', params, - [('DBSnapshot', DBSnapshot)]) - - def create_dbsnapshot(self, snapshot_id, dbinstance_id): - """ - Create a new DB snapshot. - - :type snapshot_id: string - :param snapshot_id: The identifier for the DBSnapshot - - :type dbinstance_id: string - :param dbinstance_id: The source identifier for the RDS instance from - which the snapshot is created. - - :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` - :return: The newly created DBSnapshot - """ - params = {'DBSnapshotIdentifier' : snapshot_id, - 'DBInstanceIdentifier' : dbinstance_id} - return self.get_object('CreateDBSnapshot', params, DBSnapshot) - - def delete_dbsnapshot(self, identifier): - """ - Delete a DBSnapshot - - :type identifier: string - :param identifier: The identifier of the DBSnapshot to delete - """ - params = {'DBSnapshotIdentifier' : identifier} - return self.get_object('DeleteDBSnapshot', params, DBSnapshot) - - def restore_dbinstance_from_dbsnapshot(self, identifier, instance_id, - instance_class, port=None, - availability_zone=None): - - """ - Create a new DBInstance from a DB snapshot. - - :type identifier: string - :param identifier: The identifier for the DBSnapshot - - :type instance_id: string - :param instance_id: The source identifier for the RDS instance from - which the snapshot is created. - - :type instance_class: str - :param instance_class: The compute and memory capacity of the DBInstance. - Valid values are: - db.m1.small | db.m1.large | db.m1.xlarge | - db.m2.2xlarge | db.m2.4xlarge - - :type port: int - :param port: Port number on which database accepts connections. - Valid values [1115-65535]. Defaults to 3306. - - :type availability_zone: str - :param availability_zone: Name of the availability zone to place - DBInstance into. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The newly created DBInstance - """ - params = {'DBSnapshotIdentifier' : identifier, - 'DBInstanceIdentifier' : instance_id, - 'DBInstanceClass' : instance_class} - if port: - params['Port'] = port - if availability_zone: - params['AvailabilityZone'] = availability_zone - return self.get_object('RestoreDBInstanceFromDBSnapshot', - params, DBInstance) - - def restore_dbinstance_from_point_in_time(self, source_instance_id, - target_instance_id, - use_latest=False, - restore_time=None, - dbinstance_class=None, - port=None, - availability_zone=None): - - """ - Create a new DBInstance from a point in time. - - :type source_instance_id: string - :param source_instance_id: The identifier for the source DBInstance. - - :type target_instance_id: string - :param target_instance_id: The identifier of the new DBInstance. - - :type use_latest: bool - :param use_latest: If True, the latest snapshot availabile will - be used. - - :type restore_time: datetime - :param restore_time: The date and time to restore from. Only - used if use_latest is False. - - :type instance_class: str - :param instance_class: The compute and memory capacity of the DBInstance. - Valid values are: - db.m1.small | db.m1.large | db.m1.xlarge | - db.m2.2xlarge | db.m2.4xlarge - - :type port: int - :param port: Port number on which database accepts connections. - Valid values [1115-65535]. Defaults to 3306. - - :type availability_zone: str - :param availability_zone: Name of the availability zone to place - DBInstance into. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The newly created DBInstance - """ - params = {'SourceDBInstanceIdentifier' : source_instance_id, - 'TargetDBInstanceIdentifier' : target_instance_id} - if use_latest: - params['UseLatestRestorableTime'] = 'true' - elif restore_time: - params['RestoreTime'] = restore_time.isoformat() - if dbinstance_class: - params['DBInstanceClass'] = dbinstance_class - if port: - params['Port'] = port - if availability_zone: - params['AvailabilityZone'] = availability_zone - return self.get_object('RestoreDBInstanceToPointInTime', - params, DBInstance) - - # Events - - def get_all_events(self, source_identifier=None, source_type=None, - start_time=None, end_time=None, - max_records=None, marker=None): - """ - Get information about events related to your DBInstances, - DBSecurityGroups and DBParameterGroups. - - :type source_identifier: str - :param source_identifier: If supplied, the events returned will be - limited to those that apply to the identified - source. The value of this parameter depends - on the value of source_type. If neither - parameter is specified, all events in the time - span will be returned. - - :type source_type: str - :param source_type: Specifies how the source_identifier should - be interpreted. Valid values are: - b-instance | db-security-group | - db-parameter-group | db-snapshot - - :type start_time: datetime - :param start_time: The beginning of the time interval for events. - If not supplied, all available events will - be returned. - - :type end_time: datetime - :param end_time: The ending of the time interval for events. - If not supplied, all available events will - be returned. - - :type max_records: int - :param max_records: The maximum number of records to be returned. - If more results are available, a MoreToken will - be returned in the response that can be used to - retrieve additional records. Default is 100. - - :type marker: str - :param marker: The marker provided by a previous request. - - :rtype: list - :return: A list of class:`boto.rds.event.Event` - """ - params = {} - if source_identifier and source_type: - params['SourceIdentifier'] = source_identifier - params['SourceType'] = source_type - if start_time: - params['StartTime'] = start_time.isoformat() - if end_time: - params['EndTime'] = end_time.isoformat() - if max_records: - params['MaxRecords'] = max_records - if marker: - params['Marker'] = marker - return self.get_list('DescribeEvents', params, [('Event', Event)]) - - diff --git a/backup/src/boto/rds/dbinstance.py b/backup/src/boto/rds/dbinstance.py deleted file mode 100644 index 02f9af6..0000000 --- a/backup/src/boto/rds/dbinstance.py +++ /dev/null @@ -1,264 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.rds.dbsecuritygroup import DBSecurityGroup -from boto.rds.parametergroup import ParameterGroup - -class DBInstance(object): - """ - Represents a RDS DBInstance - """ - - def __init__(self, connection=None, id=None): - self.connection = connection - self.id = id - self.create_time = None - self.engine = None - self.status = None - self.allocated_storage = None - self.endpoint = None - self.instance_class = None - self.master_username = None - self.parameter_group = None - self.security_group = None - self.availability_zone = None - self.backup_retention_period = None - self.preferred_backup_window = None - self.preferred_maintenance_window = None - self.latest_restorable_time = None - self.multi_az = False - self.pending_modified_values = None - self._in_endpoint = False - self._port = None - self._address = None - - def __repr__(self): - return 'DBInstance:%s' % self.id - - def startElement(self, name, attrs, connection): - if name == 'Endpoint': - self._in_endpoint = True - elif name == 'DBParameterGroup': - self.parameter_group = ParameterGroup(self.connection) - return self.parameter_group - elif name == 'DBSecurityGroup': - self.security_group = DBSecurityGroup(self.connection) - return self.security_group - elif name == 'PendingModifiedValues': - self.pending_modified_values = PendingModifiedValues() - return self.pending_modified_values - return None - - def endElement(self, name, value, connection): - if name == 'DBInstanceIdentifier': - self.id = value - elif name == 'DBInstanceStatus': - self.status = value - elif name == 'InstanceCreateTime': - self.create_time = value - elif name == 'Engine': - self.engine = value - elif name == 'DBInstanceStatus': - self.status = value - elif name == 'AllocatedStorage': - self.allocated_storage = int(value) - elif name == 'DBInstanceClass': - self.instance_class = value - elif name == 'MasterUsername': - self.master_username = value - elif name == 'Port': - if self._in_endpoint: - self._port = int(value) - elif name == 'Address': - if self._in_endpoint: - self._address = value - elif name == 'Endpoint': - self.endpoint = (self._address, self._port) - self._in_endpoint = False - elif name == 'AvailabilityZone': - self.availability_zone = value - elif name == 'BackupRetentionPeriod': - self.backup_retention_period = value - elif name == 'LatestRestorableTime': - self.latest_restorable_time = value - elif name == 'PreferredMaintenanceWindow': - self.preferred_maintenance_window = value - elif name == 'PreferredBackupWindow': - self.preferred_backup_window = value - elif name == 'MultiAZ': - if value.lower() == 'true': - self.multi_az = True - else: - setattr(self, name, value) - - def snapshot(self, snapshot_id): - """ - Create a new DB snapshot of this DBInstance. - - :type identifier: string - :param identifier: The identifier for the DBSnapshot - - :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` - :return: The newly created DBSnapshot - """ - return self.connection.create_dbsnapshot(snapshot_id, self.id) - - def reboot(self): - """ - Reboot this DBInstance - - :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` - :return: The newly created DBSnapshot - """ - return self.connection.reboot_dbinstance(self.id) - - def update(self, validate=False): - """ - Update the DB instance's status information by making a call to fetch - the current instance attributes from the service. - - :type validate: bool - :param validate: By default, if EC2 returns no data about the - instance the update method returns quietly. If - the validate param is True, however, it will - raise a ValueError exception if no data is - returned from EC2. - """ - rs = self.connection.get_all_dbinstances(self.id) - if len(rs) > 0: - for i in rs: - if i.id == self.id: - self.__dict__.update(i.__dict__) - elif validate: - raise ValueError('%s is not a valid Instance ID' % self.id) - return self.status - - - def stop(self, skip_final_snapshot=False, final_snapshot_id=''): - """ - Delete this DBInstance. - - :type skip_final_snapshot: bool - :param skip_final_snapshot: This parameter determines whether a final - db snapshot is created before the instance - is deleted. If True, no snapshot is created. - If False, a snapshot is created before - deleting the instance. - - :type final_snapshot_id: str - :param final_snapshot_id: If a final snapshot is requested, this - is the identifier used for that snapshot. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The deleted db instance. - """ - return self.connection.delete_dbinstance(self.id, - skip_final_snapshot, - final_snapshot_id) - - def modify(self, param_group=None, security_groups=None, - preferred_maintenance_window=None, - master_password=None, allocated_storage=None, - instance_class=None, - backup_retention_period=None, - preferred_backup_window=None, - multi_az=False, - apply_immediately=False): - """ - Modify this DBInstance. - - :type security_groups: list of str or list of DBSecurityGroup objects - :param security_groups: List of names of DBSecurityGroup to authorize on - this DBInstance. - - :type preferred_maintenance_window: str - :param preferred_maintenance_window: The weekly time range (in UTC) - during which maintenance can - occur. - Default is Sun:05:00-Sun:09:00 - - :type master_password: str - :param master_password: Password of master user for the DBInstance. - Must be 4-15 alphanumeric characters. - - :type allocated_storage: int - :param allocated_storage: The new allocated storage size, in GBs. - Valid values are [5-1024] - - :type instance_class: str - :param instance_class: The compute and memory capacity of the - DBInstance. Changes will be applied at - next maintenance window unless - apply_immediately is True. - - Valid values are: - - * db.m1.small - * db.m1.large - * db.m1.xlarge - * db.m2.xlarge - * db.m2.2xlarge - * db.m2.4xlarge - - :type apply_immediately: bool - :param apply_immediately: If true, the modifications will be applied - as soon as possible rather than waiting for - the next preferred maintenance window. - - :type backup_retention_period: int - :param backup_retention_period: The number of days for which automated - backups are retained. Setting this to - zero disables automated backups. - - :type preferred_backup_window: str - :param preferred_backup_window: The daily time range during which - automated backups are created (if - enabled). Must be in h24:mi-hh24:mi - format (UTC). - - :type multi_az: bool - :param multi_az: If True, specifies the DB Instance will be - deployed in multiple availability zones. - - :rtype: :class:`boto.rds.dbinstance.DBInstance` - :return: The modified db instance. - """ - return self.connection.modify_dbinstance(self.id, - param_group, - security_groups, - preferred_maintenance_window, - master_password, - allocated_storage, - instance_class, - backup_retention_period, - preferred_backup_window, - multi_az, - apply_immediately) - -class PendingModifiedValues(dict): - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name != 'PendingModifiedValues': - self[name] = value - diff --git a/backup/src/boto/rds/dbsecuritygroup.py b/backup/src/boto/rds/dbsecuritygroup.py deleted file mode 100644 index 1555ca0..0000000 --- a/backup/src/boto/rds/dbsecuritygroup.py +++ /dev/null @@ -1,160 +0,0 @@ -# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an DBSecurityGroup -""" -from boto.ec2.securitygroup import SecurityGroup - -class DBSecurityGroup(object): - - def __init__(self, connection=None, owner_id=None, - name=None, description=None): - self.connection = connection - self.owner_id = owner_id - self.name = name - self.description = description - self.ec2_groups = [] - self.ip_ranges = [] - - def __repr__(self): - return 'DBSecurityGroup:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'IPRange': - cidr = IPRange(self) - self.ip_ranges.append(cidr) - return cidr - elif name == 'EC2SecurityGroup': - ec2_grp = EC2SecurityGroup(self) - self.ec2_groups.append(ec2_grp) - return ec2_grp - else: - return None - - def endElement(self, name, value, connection): - if name == 'OwnerId': - self.owner_id = value - elif name == 'DBSecurityGroupName': - self.name = value - elif name == 'DBSecurityGroupDescription': - self.description = value - elif name == 'IPRanges': - pass - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_dbsecurity_group(self.name) - - def authorize(self, cidr_ip=None, ec2_group=None): - """ - Add a new rule to this DBSecurity group. - You need to pass in either a CIDR block to authorize or - and EC2 SecurityGroup. - - @type cidr_ip: string - @param cidr_ip: A valid CIDR IP range to authorize - - @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` - - @rtype: bool - @return: True if successful. - """ - if isinstance(ec2_group, SecurityGroup): - group_name = ec2_group.name - group_owner_id = ec2_group.owner_id - else: - group_name = None - group_owner_id = None - return self.connection.authorize_dbsecurity_group(self.name, - cidr_ip, - group_name, - group_owner_id) - - def revoke(self, cidr_ip=None, ec2_group=None): - """ - Revoke access to a CIDR range or EC2 SecurityGroup. - You need to pass in either a CIDR block or - an EC2 SecurityGroup from which to revoke access. - - @type cidr_ip: string - @param cidr_ip: A valid CIDR IP range to revoke - - @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` - - @rtype: bool - @return: True if successful. - """ - if isinstance(ec2_group, SecurityGroup): - group_name = ec2_group.name - group_owner_id = ec2_group.owner_id - return self.connection.revoke_dbsecurity_group( - self.name, - ec2_security_group_name=group_name, - ec2_security_group_owner_id=group_owner_id) - - # Revoking by CIDR IP range - return self.connection.revoke_dbsecurity_group( - self.name, cidr_ip=cidr_ip) - -class IPRange(object): - - def __init__(self, parent=None): - self.parent = parent - self.cidr_ip = None - self.status = None - - def __repr__(self): - return 'IPRange:%s' % self.cidr_ip - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'CIDRIP': - self.cidr_ip = value - elif name == 'Status': - self.status = value - else: - setattr(self, name, value) - -class EC2SecurityGroup(object): - - def __init__(self, parent=None): - self.parent = parent - self.name = None - self.owner_id = None - - def __repr__(self): - return 'EC2SecurityGroup:%s' % self.name - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'EC2SecurityGroupName': - self.name = value - elif name == 'EC2SecurityGroupOwnerId': - self.owner_id = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/rds/dbsnapshot.py b/backup/src/boto/rds/dbsnapshot.py deleted file mode 100644 index 78d0230..0000000 --- a/backup/src/boto/rds/dbsnapshot.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class DBSnapshot(object): - """ - Represents a RDS DB Snapshot - """ - - def __init__(self, connection=None, id=None): - self.connection = connection - self.id = id - self.engine = None - self.snapshot_create_time = None - self.instance_create_time = None - self.port = None - self.status = None - self.availability_zone = None - self.master_username = None - self.allocated_storage = None - self.instance_id = None - self.availability_zone = None - - def __repr__(self): - return 'DBSnapshot:%s' % self.id - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'Engine': - self.engine = value - elif name == 'InstanceCreateTime': - self.instance_create_time = value - elif name == 'SnapshotCreateTime': - self.snapshot_create_time = value - elif name == 'DBInstanceIdentifier': - self.instance_id = value - elif name == 'DBSnapshotIdentifier': - self.id = value - elif name == 'Port': - self.port = int(value) - elif name == 'Status': - self.status = value - elif name == 'AvailabilityZone': - self.availability_zone = value - elif name == 'MasterUsername': - self.master_username = value - elif name == 'AllocatedStorage': - self.allocated_storage = int(value) - elif name == 'SnapshotTime': - self.time = value - else: - setattr(self, name, value) - - - diff --git a/backup/src/boto/rds/event.py b/backup/src/boto/rds/event.py deleted file mode 100644 index a91f8f0..0000000 --- a/backup/src/boto/rds/event.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Event(object): - - def __init__(self, connection=None): - self.connection = connection - self.message = None - self.source_identifier = None - self.source_type = None - self.engine = None - self.date = None - - def __repr__(self): - return '"%s"' % self.message - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'SourceIdentifier': - self.source_identifier = value - elif name == 'SourceType': - self.source_type = value - elif name == 'Message': - self.message = value - elif name == 'Date': - self.date = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/rds/parametergroup.py b/backup/src/boto/rds/parametergroup.py deleted file mode 100644 index 44d00e2..0000000 --- a/backup/src/boto/rds/parametergroup.py +++ /dev/null @@ -1,201 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class ParameterGroup(dict): - - def __init__(self, connection=None): - dict.__init__(self) - self.connection = connection - self.name = None - self.description = None - self.engine = None - self._current_param = None - - def __repr__(self): - return 'ParameterGroup:%s' % self.name - - def startElement(self, name, attrs, connection): - if name == 'Parameter': - if self._current_param: - self[self._current_param.name] = self._current_param - self._current_param = Parameter(self) - return self._current_param - - def endElement(self, name, value, connection): - if name == 'DBParameterGroupName': - self.name = value - elif name == 'Description': - self.description = value - elif name == 'Engine': - self.engine = value - else: - setattr(self, name, value) - - def modifiable(self): - mod = [] - for key in self: - p = self[key] - if p.is_modifiable: - mod.append(p) - return mod - - def get_params(self): - pg = self.connection.get_all_dbparameters(self.name) - self.update(pg) - - def add_param(self, name, value, apply_method): - param = Parameter() - param.name = name - param.value = value - param.apply_method = apply_method - self.params.append(param) - -class Parameter(object): - """ - Represents a RDS Parameter - """ - - ValidTypes = {'integer' : int, - 'string' : str, - 'boolean' : bool} - ValidSources = ['user', 'system', 'engine-default'] - ValidApplyTypes = ['static', 'dynamic'] - ValidApplyMethods = ['immediate', 'pending-reboot'] - - def __init__(self, group=None, name=None): - self.group = group - self.name = name - self._value = None - self.type = str - self.source = None - self.is_modifiable = True - self.description = None - self.apply_method = None - self.allowed_values = None - - def __repr__(self): - return 'Parameter:%s' % self.name - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'ParameterName': - self.name = value - elif name == 'ParameterValue': - self._value = value - elif name == 'DataType': - if value in self.ValidTypes: - self.type = value - elif name == 'Source': - if value in self.ValidSources: - self.source = value - elif name == 'IsModifiable': - if value.lower() == 'true': - self.is_modifiable = True - else: - self.is_modifiable = False - elif name == 'Description': - self.description = value - elif name == 'ApplyType': - if value in self.ValidApplyTypes: - self.apply_type = value - elif name == 'AllowedValues': - self.allowed_values = value - else: - setattr(self, name, value) - - def merge(self, d, i): - prefix = 'Parameters.member.%d.' % i - if self.name: - d[prefix+'ParameterName'] = self.name - if self._value: - d[prefix+'ParameterValue'] = self._value - if self.apply_type: - d[prefix+'ApplyMethod'] = self.apply_method - - def _set_string_value(self, value): - if not isinstance(value, str) or isinstance(value, unicode): - raise ValueError, 'value must be of type str' - if self.allowed_values: - choices = self.allowed_values.split(',') - if value not in choices: - raise ValueError, 'value must be in %s' % self.allowed_values - self._value = value - - def _set_integer_value(self, value): - if isinstance(value, str) or isinstance(value, unicode): - value = int(value) - if isinstance(value, int) or isinstance(value, long): - if self.allowed_values: - min, max = self.allowed_values.split('-') - if value < int(min) or value > int(max): - raise ValueError, 'range is %s' % self.allowed_values - self._value = value - else: - raise ValueError, 'value must be integer' - - def _set_boolean_value(self, value): - if isinstance(value, bool): - self._value = value - elif isinstance(value, str) or isinstance(value, unicode): - if value.lower() == 'true': - self._value = True - else: - self._value = False - else: - raise ValueError, 'value must be boolean' - - def set_value(self, value): - if self.type == 'string': - self._set_string_value(value) - elif self.type == 'integer': - self._set_integer_value(value) - elif self.type == 'boolean': - self._set_boolean_value(value) - else: - raise TypeError, 'unknown type (%s)' % self.type - - def get_value(self): - if self._value == None: - return self._value - if self.type == 'string': - return self._value - elif self.type == 'integer': - if not isinstance(self._value, int) and not isinstance(self._value, long): - self._set_integer_value(self._value) - return self._value - elif self.type == 'boolean': - if not isinstance(self._value, bool): - self._set_boolean_value(self._value) - return self._value - else: - raise TypeError, 'unknown type (%s)' % self.type - - value = property(get_value, set_value, 'The value of the parameter') - - def apply(self, immediate=False): - if immediate: - self.apply_method = 'immediate' - else: - self.apply_method = 'pending-reboot' - self.group.connection.modify_parameter_group(self.group.name, [self]) - diff --git a/backup/src/boto/rds/regioninfo.py b/backup/src/boto/rds/regioninfo.py deleted file mode 100644 index 7d186ae..0000000 --- a/backup/src/boto/rds/regioninfo.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from boto.regioninfo import RegionInfo - -class RDSRegionInfo(RegionInfo): - - def __init__(self, connection=None, name=None, endpoint=None): - from boto.rds import RDSConnection - RegionInfo.__init__(self, connection, name, endpoint, - RDSConnection) diff --git a/backup/src/boto/regioninfo.py b/backup/src/boto/regioninfo.py deleted file mode 100644 index 907385f..0000000 --- a/backup/src/boto/regioninfo.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class RegionInfo(object): - """ - Represents an AWS Region - """ - - def __init__(self, connection=None, name=None, endpoint=None, - connection_cls=None): - self.connection = connection - self.name = name - self.endpoint = endpoint - self.connection_cls = connection_cls - - def __repr__(self): - return 'RegionInfo:%s' % self.name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'regionName': - self.name = value - elif name == 'regionEndpoint': - self.endpoint = value - else: - setattr(self, name, value) - - def connect(self, **kw_params): - """ - Connect to this Region's endpoint. Returns an connection - object pointing to the endpoint associated with this region. - You may pass any of the arguments accepted by the connection - class's constructor as keyword arguments and they will be - passed along to the connection object. - - :rtype: Connection object - :return: The connection to this regions endpoint - """ - if self.connection_cls: - return self.connection_cls(region=self, **kw_params) - - diff --git a/backup/src/boto/resultset.py b/backup/src/boto/resultset.py deleted file mode 100644 index 075fc5e..0000000 --- a/backup/src/boto/resultset.py +++ /dev/null @@ -1,153 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class ResultSet(list): - """ - The ResultSet is used to pass results back from the Amazon services - to the client. It is light wrapper around Python's :py:class:`list` class, - with some additional methods for parsing XML results from AWS. - Because I don't really want any dependencies on external libraries, - I'm using the standard SAX parser that comes with Python. The good news is - that it's quite fast and efficient but it makes some things rather - difficult. - - You can pass in, as the marker_elem parameter, a list of tuples. - Each tuple contains a string as the first element which represents - the XML element that the resultset needs to be on the lookout for - and a Python class as the second element of the tuple. Each time the - specified element is found in the XML, a new instance of the class - will be created and popped onto the stack. - - :ivar str next_token: A hash used to assist in paging through very long - result sets. In most cases, passing this value to certain methods - will give you another 'page' of results. - """ - def __init__(self, marker_elem=None): - list.__init__(self) - if isinstance(marker_elem, list): - self.markers = marker_elem - else: - self.markers = [] - self.marker = None - self.key_marker = None - self.next_key_marker = None - self.next_version_id_marker = None - self.version_id_marker = None - self.is_truncated = False - self.next_token = None - self.status = True - - def startElement(self, name, attrs, connection): - for t in self.markers: - if name == t[0]: - obj = t[1](connection) - self.append(obj) - return obj - return None - - def to_boolean(self, value, true_value='true'): - if value == true_value: - return True - else: - return False - - def endElement(self, name, value, connection): - if name == 'IsTruncated': - self.is_truncated = self.to_boolean(value) - elif name == 'Marker': - self.marker = value - elif name == 'KeyMarker': - self.key_marker = value - elif name == 'NextKeyMarker': - self.next_key_marker = value - elif name == 'VersionIdMarker': - self.version_id_marker = value - elif name == 'NextVersionIdMarker': - self.next_version_id_marker = value - elif name == 'UploadIdMarker': - self.upload_id_marker = value - elif name == 'NextUploadIdMarker': - self.next_upload_id_marker = value - elif name == 'Bucket': - self.bucket = value - elif name == 'MaxUploads': - self.max_uploads = int(value) - elif name == 'Prefix': - self.prefix = value - elif name == 'return': - self.status = self.to_boolean(value) - elif name == 'StatusCode': - self.status = self.to_boolean(value, 'Success') - elif name == 'ItemName': - self.append(value) - elif name == 'NextToken': - self.next_token = value - elif name == 'BoxUsage': - try: - connection.box_usage += float(value) - except: - pass - elif name == 'IsValid': - self.status = self.to_boolean(value, 'True') - else: - setattr(self, name, value) - -class BooleanResult(object): - - def __init__(self, marker_elem=None): - self.status = True - self.request_id = None - self.box_usage = None - - def __repr__(self): - if self.status: - return 'True' - else: - return 'False' - - def __nonzero__(self): - return self.status - - def startElement(self, name, attrs, connection): - return None - - def to_boolean(self, value, true_value='true'): - if value == true_value: - return True - else: - return False - - def endElement(self, name, value, connection): - if name == 'return': - self.status = self.to_boolean(value) - elif name == 'StatusCode': - self.status = self.to_boolean(value, 'Success') - elif name == 'IsValid': - self.status = self.to_boolean(value, 'True') - elif name == 'RequestId': - self.request_id = value - elif name == 'requestId': - self.request_id = value - elif name == 'BoxUsage': - self.request_id = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/route53/__init__.py b/backup/src/boto/route53/__init__.py deleted file mode 100644 index d404bc7..0000000 --- a/backup/src/boto/route53/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# this is here for backward compatibility -# originally, the Route53Connection class was defined here -from connection import Route53Connection diff --git a/backup/src/boto/route53/connection.py b/backup/src/boto/route53/connection.py deleted file mode 100644 index bbd218c..0000000 --- a/backup/src/boto/route53/connection.py +++ /dev/null @@ -1,285 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -import xml.sax -import time -import uuid -import urllib -import boto -from boto.connection import AWSAuthConnection -from boto import handler -from boto.resultset import ResultSet -import boto.jsonresponse -import exception -import hostedzone - -HZXML = """ - - %(name)s - %(caller_ref)s - - %(comment)s - -""" - -#boto.set_stream_logger('dns') - -class Route53Connection(AWSAuthConnection): - - DefaultHost = 'route53.amazonaws.com' - Version = '2010-10-01' - XMLNameSpace = 'https://route53.amazonaws.com/doc/2010-10-01/' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - port=None, proxy=None, proxy_port=None, - host=DefaultHost, debug=0): - AWSAuthConnection.__init__(self, host, - aws_access_key_id, aws_secret_access_key, - True, port, proxy, proxy_port, debug=debug) - - def _required_auth_capability(self): - return ['route53'] - - def make_request(self, action, path, headers=None, data='', params=None): - if params: - pairs = [] - for key, val in params.iteritems(): - if val is None: continue - pairs.append(key + '=' + urllib.quote(str(val))) - path += '?' + '&'.join(pairs) - return AWSAuthConnection.make_request(self, action, path, headers, data) - - # Hosted Zones - - def get_all_hosted_zones(self): - """ - Returns a Python data structure with information about all - Hosted Zones defined for the AWS account. - """ - response = self.make_request('GET', '/%s/hostedzone' % self.Version) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise exception.DNSServerError(response.status, - response.reason, - body) - e = boto.jsonresponse.Element(list_marker='HostedZones', - item_marker=('HostedZone',)) - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - - def get_hosted_zone(self, hosted_zone_id): - """ - Get detailed information about a particular Hosted Zone. - - :type hosted_zone_id: str - :param hosted_zone_id: The unique identifier for the Hosted Zone - - """ - uri = '/%s/hostedzone/%s' % (self.Version, hosted_zone_id) - response = self.make_request('GET', uri) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise exception.DNSServerError(response.status, - response.reason, - body) - e = boto.jsonresponse.Element(list_marker='NameServers', - item_marker=('NameServer',)) - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - - def create_hosted_zone(self, domain_name, caller_ref=None, comment=''): - """ - Create a new Hosted Zone. Returns a Python data structure with - information about the newly created Hosted Zone. - - :type domain_name: str - :param domain_name: The name of the domain. This should be a - fully-specified domain, and should end with - a final period as the last label indication. - If you omit the final period, Amazon Route 53 - assumes the domain is relative to the root. - This is the name you have registered with your - DNS registrar. It is also the name you will - delegate from your registrar to the Amazon - Route 53 delegation servers returned in - response to this request.A list of strings - with the image IDs wanted - - :type caller_ref: str - :param caller_ref: A unique string that identifies the request - and that allows failed CreateHostedZone requests - to be retried without the risk of executing the - operation twice. - If you don't provide a value for this, boto will - generate a Type 4 UUID and use that. - - :type comment: str - :param comment: Any comments you want to include about the hosted - zone. - - """ - if caller_ref is None: - caller_ref = str(uuid.uuid4()) - params = {'name' : domain_name, - 'caller_ref' : caller_ref, - 'comment' : comment, - 'xmlns' : self.XMLNameSpace} - xml = HZXML % params - uri = '/%s/hostedzone' % self.Version - response = self.make_request('POST', uri, - {'Content-Type' : 'text/xml'}, xml) - body = response.read() - boto.log.debug(body) - if response.status == 201: - e = boto.jsonresponse.Element(list_marker='NameServers', - item_marker=('NameServer',)) - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - else: - raise exception.DNSServerError(response.status, - response.reason, - body) - - def delete_hosted_zone(self, hosted_zone_id): - uri = '/%s/hostedzone/%s' % (self.Version, hosted_zone_id) - response = self.make_request('DELETE', uri) - body = response.read() - boto.log.debug(body) - if response.status not in (200, 204): - raise exception.DNSServerError(response.status, - response.reason, - body) - e = boto.jsonresponse.Element() - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - - # Resource Record Sets - - def get_all_rrsets(self, hosted_zone_id, type=None, - name=None, maxitems=None): - """ - Retrieve the Resource Record Sets defined for this Hosted Zone. - Returns the raw XML data returned by the Route53 call. - - :type hosted_zone_id: str - :param hosted_zone_id: The unique identifier for the Hosted Zone - - :type type: str - :param type: The type of resource record set to begin the record - listing from. Valid choices are: - - * A - * AAAA - * CNAME - * MX - * NS - * PTR - * SOA - * SPF - * SRV - * TXT - - :type name: str - :param name: The first name in the lexicographic ordering of domain - names to be retrieved - - :type maxitems: int - :param maxitems: The maximum number of records - - """ - from boto.route53.record import ResourceRecordSets - params = {'type': type, 'name': name, 'maxitems': maxitems} - uri = '/%s/hostedzone/%s/rrset' % (self.Version, hosted_zone_id) - response = self.make_request('GET', uri, params=params) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise exception.DNSServerError(response.status, - response.reason, - body) - rs = ResourceRecordSets(connection=self, hosted_zone_id=hosted_zone_id) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - - def change_rrsets(self, hosted_zone_id, xml_body): - """ - Create or change the authoritative DNS information for this - Hosted Zone. - Returns a Python data structure with information about the set of - changes, including the Change ID. - - :type hosted_zone_id: str - :param hosted_zone_id: The unique identifier for the Hosted Zone - - :type xml_body: str - :param xml_body: The list of changes to be made, defined in the - XML schema defined by the Route53 service. - - """ - uri = '/%s/hostedzone/%s/rrset' % (self.Version, hosted_zone_id) - response = self.make_request('POST', uri, - {'Content-Type' : 'text/xml'}, - xml_body) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise exception.DNSServerError(response.status, - response.reason, - body) - e = boto.jsonresponse.Element() - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - - def get_change(self, change_id): - """ - Get information about a proposed set of changes, as submitted - by the change_rrsets method. - Returns a Python data structure with status information about the - changes. - - :type change_id: str - :param change_id: The unique identifier for the set of changes. - This ID is returned in the response to the - change_rrsets method. - - """ - uri = '/%s/change/%s' % (self.Version, change_id) - response = self.make_request('GET', uri) - body = response.read() - boto.log.debug(body) - if response.status >= 300: - raise exception.DNSServerError(response.status, - response.reason, - body) - e = boto.jsonresponse.Element() - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e diff --git a/backup/src/boto/route53/exception.py b/backup/src/boto/route53/exception.py deleted file mode 100644 index ba41285..0000000 --- a/backup/src/boto/route53/exception.py +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.exception import BotoServerError - -class DNSServerError(BotoServerError): - - pass diff --git a/backup/src/boto/route53/hostedzone.py b/backup/src/boto/route53/hostedzone.py deleted file mode 100644 index 66b79b8..0000000 --- a/backup/src/boto/route53/hostedzone.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -class HostedZone(object): - - def __init__(self, id=None, name=None, owner=None, version=None, - caller_reference=None, config=None): - self.id = id - self.name = name - self.owner = owner - self.version = version - self.caller_reference = caller_reference - self.config = config - - def startElement(self, name, attrs, connection): - if name == 'Config': - self.config = Config() - return self.config - else: - return None - - def endElement(self, name, value, connection): - if name == 'Id': - self.id = value - elif name == 'Name': - self.name = value - elif name == 'Owner': - self.owner = value - elif name == 'Version': - self.version = value - elif name == 'CallerReference': - self.caller_reference = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/route53/record.py b/backup/src/boto/route53/record.py deleted file mode 100644 index 24f0482..0000000 --- a/backup/src/boto/route53/record.py +++ /dev/null @@ -1,152 +0,0 @@ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -RECORD_TYPES = ['A', 'AAAA', 'TXT', 'CNAME', 'MX', 'PTR', 'SRV', 'SPF'] - -from boto.resultset import ResultSet -class ResourceRecordSets(ResultSet): - - ChangeResourceRecordSetsBody = """ - - - %(comment)s - %(changes)s - - """ - - ChangeXML = """ - %(action)s - %(record)s - """ - - - def __init__(self, connection=None, hosted_zone_id=None, comment=None): - self.connection = connection - self.hosted_zone_id = hosted_zone_id - self.comment = comment - self.changes = [] - self.next_record_name = None - self.next_record_type = None - ResultSet.__init__(self, [('ResourceRecordSet', Record)]) - - def __repr__(self): - return '' % self.hosted_zone_id - - def add_change(self, action, name, type, ttl=600): - """Add a change request""" - change = Record(name, type, ttl) - self.changes.append([action, change]) - return change - - def to_xml(self): - """Convert this ResourceRecordSet into XML - to be saved via the ChangeResourceRecordSetsRequest""" - changesXML = "" - for change in self.changes: - changeParams = {"action": change[0], "record": change[1].to_xml()} - changesXML += self.ChangeXML % changeParams - params = {"comment": self.comment, "changes": changesXML} - return self.ChangeResourceRecordSetsBody % params - - def commit(self): - """Commit this change""" - if not self.connection: - import boto - self.connection = boto.connect_route53() - return self.connection.change_rrsets(self.hosted_zone_id, self.to_xml()) - - def endElement(self, name, value, connection): - """Overwritten to also add the NextRecordName and - NextRecordType to the base object""" - if name == 'NextRecordName': - self.next_record_name = value - elif name == 'NextRecordType': - self.next_record_type = value - else: - return ResultSet.endElement(self, name, value, connection) - - def __iter__(self): - """Override the next function to support paging""" - results = ResultSet.__iter__(self) - while results: - for obj in results: - yield obj - if self.is_truncated: - self.is_truncated = False - results = self.connection.get_all_rrsets(self.hosted_zone_id, name=self.next_record_name, type=self.next_record_type) - else: - results = None - - - -class Record(object): - """An individual ResourceRecordSet""" - - XMLBody = """ - %(name)s - %(type)s - %(ttl)s - %(records)s - """ - - ResourceRecordBody = """ - %s - """ - - - def __init__(self, name=None, type=None, ttl=600, resource_records=None): - self.name = name - self.type = type - self.ttl = ttl - if resource_records == None: - resource_records = [] - self.resource_records = resource_records - - def add_value(self, value): - """Add a resource record value""" - self.resource_records.append(value) - - def to_xml(self): - """Spit this resource record set out as XML""" - records = "" - for r in self.resource_records: - records += self.ResourceRecordBody % r - params = { - "name": self.name, - "type": self.type, - "ttl": self.ttl, - "records": records - } - return self.XMLBody % params - - def endElement(self, name, value, connection): - if name == 'Name': - self.name = value - elif name == 'Type': - self.type = value - elif name == 'TTL': - self.ttl = value - elif name == 'Value': - self.resource_records.append(value) - - def startElement(self, name, attrs, connection): - return None diff --git a/backup/src/boto/s3/__init__.py b/backup/src/boto/s3/__init__.py deleted file mode 100644 index f3f4c1e..0000000 --- a/backup/src/boto/s3/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/backup/src/boto/s3/acl.py b/backup/src/boto/s3/acl.py deleted file mode 100644 index 2640499..0000000 --- a/backup/src/boto/s3/acl.py +++ /dev/null @@ -1,163 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.s3.user import User - - -CannedACLStrings = ['private', 'public-read', - 'public-read-write', 'authenticated-read', - 'bucket-owner-read', 'bucket-owner-full-control'] - - -class Policy: - - def __init__(self, parent=None): - self.parent = parent - self.acl = None - - def __repr__(self): - grants = [] - for g in self.acl.grants: - if g.id == self.owner.id: - grants.append("%s (owner) = %s" % (g.display_name, g.permission)) - else: - if g.type == 'CanonicalUser': - u = g.display_name - elif g.type == 'Group': - u = g.uri - else: - u = g.email - grants.append("%s = %s" % (u, g.permission)) - return "" % ", ".join(grants) - - def startElement(self, name, attrs, connection): - if name == 'Owner': - self.owner = User(self) - return self.owner - elif name == 'AccessControlList': - self.acl = ACL(self) - return self.acl - else: - return None - - def endElement(self, name, value, connection): - if name == 'Owner': - pass - elif name == 'AccessControlList': - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '' - s += self.owner.to_xml() - s += self.acl.to_xml() - s += '' - return s - -class ACL: - - def __init__(self, policy=None): - self.policy = policy - self.grants = [] - - def add_grant(self, grant): - self.grants.append(grant) - - def add_email_grant(self, permission, email_address): - grant = Grant(permission=permission, type='AmazonCustomerByEmail', - email_address=email_address) - self.grants.append(grant) - - def add_user_grant(self, permission, user_id): - grant = Grant(permission=permission, type='CanonicalUser', id=user_id) - self.grants.append(grant) - - def startElement(self, name, attrs, connection): - if name == 'Grant': - self.grants.append(Grant(self)) - return self.grants[-1] - else: - return None - - def endElement(self, name, value, connection): - if name == 'Grant': - pass - else: - setattr(self, name, value) - - def to_xml(self): - s = '' - for grant in self.grants: - s += grant.to_xml() - s += '' - return s - -class Grant: - - NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' - - def __init__(self, permission=None, type=None, id=None, - display_name=None, uri=None, email_address=None): - self.permission = permission - self.id = id - self.display_name = display_name - self.uri = uri - self.email_address = email_address - self.type = type - - def startElement(self, name, attrs, connection): - if name == 'Grantee': - self.type = attrs['xsi:type'] - return None - - def endElement(self, name, value, connection): - if name == 'ID': - self.id = value - elif name == 'DisplayName': - self.display_name = value - elif name == 'URI': - self.uri = value - elif name == 'EmailAddress': - self.email_address = value - elif name == 'Grantee': - pass - elif name == 'Permission': - self.permission = value - else: - setattr(self, name, value) - - def to_xml(self): - s = '' - s += '' % (self.NameSpace, self.type) - if self.type == 'CanonicalUser': - s += '%s' % self.id - s += '%s' % self.display_name - elif self.type == 'Group': - s += '%s' % self.uri - else: - s += '%s' % self.email_address - s += '' - s += '%s' % self.permission - s += '' - return s - - diff --git a/backup/src/boto/s3/bucket.py b/backup/src/boto/s3/bucket.py deleted file mode 100644 index 6188196..0000000 --- a/backup/src/boto/s3/bucket.py +++ /dev/null @@ -1,1068 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto import handler -from boto.provider import Provider -from boto.resultset import ResultSet -from boto.s3.acl import ACL, Policy, CannedACLStrings, Grant -from boto.s3.key import Key -from boto.s3.prefix import Prefix -from boto.s3.deletemarker import DeleteMarker -from boto.s3.user import User -from boto.s3.multipart import MultiPartUpload -from boto.s3.multipart import CompleteMultiPartUpload -from boto.s3.bucketlistresultset import BucketListResultSet -from boto.s3.bucketlistresultset import VersionedBucketListResultSet -from boto.s3.bucketlistresultset import MultiPartUploadListResultSet -import boto.jsonresponse -import boto.utils -import xml.sax -import urllib -import re -from collections import defaultdict - -# as per http://goo.gl/BDuud (02/19/2011) -class S3WebsiteEndpointTranslate: - trans_region = defaultdict(lambda :'s3-website-us-east-1') - - trans_region['EU'] = 's3-website-eu-west-1' - trans_region['us-west-1'] = 's3-website-us-west-1' - trans_region['ap-southeast-1'] = 's3-website-ap-southeast-1' - - @classmethod - def translate_region(self, reg): - return self.trans_region[reg] - -S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] - -class Bucket(object): - - BucketLoggingBody = """ - - - %s - %s - - """ - - EmptyBucketLoggingBody = """ - - """ - - LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' - - BucketPaymentBody = """ - - %s - """ - - VersioningBody = """ - - %s - %s - """ - - WebsiteBody = """ - - %s - %s - """ - - WebsiteErrorFragment = """%s""" - - VersionRE = '([A-Za-z]+)' - MFADeleteRE = '([A-Za-z]+)' - - def __init__(self, connection=None, name=None, key_class=Key): - self.name = name - self.connection = connection - self.key_class = key_class - - def __repr__(self): - return '' % self.name - - def __iter__(self): - return iter(BucketListResultSet(self)) - - def __contains__(self, key_name): - return not (self.get_key(key_name) is None) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Name': - self.name = value - elif name == 'CreationDate': - self.creation_date = value - else: - setattr(self, name, value) - - def set_key_class(self, key_class): - """ - Set the Key class associated with this bucket. By default, this - would be the boto.s3.key.Key class but if you want to subclass that - for some reason this allows you to associate your new class with a - bucket so that when you call bucket.new_key() or when you get a listing - of keys in the bucket you will get an instances of your key class - rather than the default. - - :type key_class: class - :param key_class: A subclass of Key that can be more specific - """ - self.key_class = key_class - - def lookup(self, key_name, headers=None): - """ - Deprecated: Please use get_key method. - - :type key_name: string - :param key_name: The name of the key to retrieve - - :rtype: :class:`boto.s3.key.Key` - :returns: A Key object from this bucket. - """ - return self.get_key(key_name, headers=headers) - - def get_key(self, key_name, headers=None, version_id=None): - """ - Check to see if a particular key exists within the bucket. This - method uses a HEAD request to check for the existance of the key. - Returns: An instance of a Key object or None - - :type key_name: string - :param key_name: The name of the key to retrieve - - :rtype: :class:`boto.s3.key.Key` - :returns: A Key object from this bucket. - """ - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - response = self.connection.make_request('HEAD', self.name, key_name, - headers=headers, - query_args=query_args) - # Allow any success status (2xx) - for example this lets us - # support Range gets, which return status 206: - if response.status/100 == 2: - response.read() - k = self.key_class(self) - provider = self.connection.provider - k.metadata = boto.utils.get_aws_metadata(response.msg, provider) - k.etag = response.getheader('etag') - k.content_type = response.getheader('content-type') - k.content_encoding = response.getheader('content-encoding') - k.last_modified = response.getheader('last-modified') - k.size = int(response.getheader('content-length')) - k.cache_control = response.getheader('cache-control') - k.name = key_name - k.handle_version_headers(response) - return k - else: - if response.status == 404: - response.read() - return None - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, '') - - def list(self, prefix='', delimiter='', marker='', headers=None): - """ - List key objects within a bucket. This returns an instance of an - BucketListResultSet that automatically handles all of the result - paging, etc. from S3. You just need to keep iterating until - there are no more results. - - Called with no arguments, this will return an iterator object across - all keys within the bucket. - - The Key objects returned by the iterator are obtained by parsing - the results of a GET on the bucket, also known as the List Objects - request. The XML returned by this request contains only a subset - of the information about each key. Certain metadata fields such - as Content-Type and user metadata are not available in the XML. - Therefore, if you want these additional metadata fields you will - have to do a HEAD request on the Key in the bucket. - - :type prefix: string - :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. - - :type delimiter: string - :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - for more details. - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return BucketListResultSet(self, prefix, delimiter, marker, headers) - - def list_versions(self, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): - """ - List version objects within a bucket. This returns an instance of an - VersionedBucketListResultSet that automatically handles all of the result - paging, etc. from S3. You just need to keep iterating until - there are no more results. - Called with no arguments, this will return an iterator object across - all keys within the bucket. - - :type prefix: string - :param prefix: allows you to limit the listing to a particular - prefix. For example, if you call the method with - prefix='/foo/' then the iterator will only cycle - through the keys that begin with the string '/foo/'. - - :type delimiter: string - :param delimiter: can be used in conjunction with the prefix - to allow you to organize and browse your keys - hierarchically. See: - http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ - for more details. - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return VersionedBucketListResultSet(self, prefix, delimiter, key_marker, - version_id_marker, headers) - - def list_multipart_uploads(self, key_marker='', - upload_id_marker='', - headers=None): - """ - List multipart upload objects within a bucket. This returns an - instance of an MultiPartUploadListResultSet that automatically - handles all of the result paging, etc. from S3. You just need - to keep iterating until there are no more results. - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` - :return: an instance of a BucketListResultSet that handles paging, etc - """ - return MultiPartUploadListResultSet(self, key_marker, - upload_id_marker, - headers) - - def _get_all(self, element_map, initial_query_string='', - headers=None, **params): - l = [] - for k,v in params.items(): - k = k.replace('_', '-') - if k == 'maxkeys': - k = 'max-keys' - if isinstance(v, unicode): - v = v.encode('utf-8') - if v is not None and v != '': - l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) - if len(l): - s = initial_query_string + '&' + '&'.join(l) - else: - s = initial_query_string - response = self.connection.make_request('GET', self.name, - headers=headers, query_args=s) - body = response.read() - boto.log.debug(body) - if response.status == 200: - rs = ResultSet(element_map) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_all_keys(self, headers=None, **params): - """ - A lower-level method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type marker: string - :param marker: The "marker" of where you are in the result set - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Contents', self.key_class), - ('CommonPrefixes', Prefix)], - '', headers, **params) - - def get_all_versions(self, headers=None, **params): - """ - A lower-level, version-aware method for listing contents of a bucket. - This closely models the actual S3 API and requires you to manually - handle the paging of results. For a higher-level method - that handles the details of paging for you, you can use the list method. - - :type max_keys: int - :param max_keys: The maximum number of keys to retrieve - - :type prefix: string - :param prefix: The prefix of the keys you want to retrieve - - :type key_marker: string - :param key_marker: The "marker" of where you are in the result set - with respect to keys. - - :type version_id_marker: string - :param version_id_marker: The "marker" of where you are in the result - set with respect to version-id's. - - :type delimiter: string - :param delimiter: If this optional, Unicode string parameter - is included with your request, then keys that - contain the same string between the prefix and - the first occurrence of the delimiter will be - rolled up into a single result element in the - CommonPrefixes collection. These rolled-up keys - are not returned elsewhere in the response. - - :rtype: ResultSet - :return: The result from S3 listing the keys requested - - """ - return self._get_all([('Version', self.key_class), - ('CommonPrefixes', Prefix), - ('DeleteMarker', DeleteMarker)], - 'versions', headers, **params) - - def get_all_multipart_uploads(self, headers=None, **params): - """ - A lower-level, version-aware method for listing active - MultiPart uploads for a bucket. This closely models the - actual S3 API and requires you to manually handle the paging - of results. For a higher-level method that handles the - details of paging for you, you can use the list method. - - :type max_uploads: int - :param max_uploads: The maximum number of uploads to retrieve. - Default value is 1000. - - :type key_marker: string - :param key_marker: Together with upload_id_marker, this parameter - specifies the multipart upload after which listing - should begin. If upload_id_marker is not specified, - only the keys lexicographically greater than the - specified key_marker will be included in the list. - - If upload_id_marker is specified, any multipart - uploads for a key equal to the key_marker might - also be included, provided those multipart uploads - have upload IDs lexicographically greater than the - specified upload_id_marker. - - :type upload_id_marker: string - :param upload_id_marker: Together with key-marker, specifies - the multipart upload after which listing - should begin. If key_marker is not specified, - the upload_id_marker parameter is ignored. - Otherwise, any multipart uploads for a key - equal to the key_marker might be included - in the list only if they have an upload ID - lexicographically greater than the specified - upload_id_marker. - - - :rtype: ResultSet - :return: The result from S3 listing the uploads requested - - """ - return self._get_all([('Upload', MultiPartUpload)], - 'uploads', headers, **params) - - def new_key(self, key_name=None): - """ - Creates a new key - - :type key_name: string - :param key_name: The name of the key to create - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - return self.key_class(self, key_name) - - def generate_url(self, expires_in, method='GET', - headers=None, force_http=False): - return self.connection.generate_url(expires_in, method, self.name, - headers=headers, - force_http=force_http) - - def delete_key(self, key_name, headers=None, - version_id=None, mfa_token=None): - """ - Deletes a key from the bucket. If a version_id is provided, - only that version of the key will be deleted. - - :type key_name: string - :param key_name: The key name to delete - - :type version_id: string - :param version_id: The version ID (optional) - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required anytime you are - deleting versioned objects from a bucket - that has the MFADelete option on the bucket. - """ - provider = self.connection.provider - if version_id: - query_args = 'versionId=%s' % version_id - else: - query_args = None - if mfa_token: - if not headers: - headers = {} - headers[provider.mfa_header] = ' '.join(mfa_token) - response = self.connection.make_request('DELETE', self.name, key_name, - headers=headers, - query_args=query_args) - body = response.read() - if response.status != 204: - raise provider.storage_response_error(response.status, - response.reason, body) - - def copy_key(self, new_key_name, src_bucket_name, - src_key_name, metadata=None, src_version_id=None, - storage_class='STANDARD', preserve_acl=False): - """ - Create a new key in the bucket by copying another existing key. - - :type new_key_name: string - :param new_key_name: The name of the new key - - :type src_bucket_name: string - :param src_bucket_name: The name of the source bucket - - :type src_key_name: string - :param src_key_name: The name of the source key - - :type src_version_id: string - :param src_version_id: The version id for the key. This param - is optional. If not specified, the newest - version of the key will be copied. - - :type metadata: dict - :param metadata: Metadata to be associated with new key. - If metadata is supplied, it will replace the - metadata of the source key being copied. - If no metadata is supplied, the source key's - metadata will be copied to the new key. - - :type storage_class: string - :param storage_class: The storage class of the new key. - By default, the new key will use the - standard storage class. Possible values are: - STANDARD | REDUCED_REDUNDANCY - - :type preserve_acl: bool - :param preserve_acl: If True, the ACL from the source key - will be copied to the destination - key. If False, the destination key - will have the default ACL. - Note that preserving the ACL in the - new key object will require two - additional API calls to S3, one to - retrieve the current ACL and one to - set that ACL on the new object. If - you don't care about the ACL, a value - of False will be significantly more - efficient. - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - if preserve_acl: - acl = self.get_xml_acl(src_key_name) - src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) - if src_version_id: - src += '?version_id=%s' % src_version_id - provider = self.connection.provider - headers = {provider.copy_source_header : src} - if storage_class != 'STANDARD': - headers[provider.storage_class_header] = storage_class - if metadata: - headers[provider.metadata_directive_header] = 'REPLACE' - headers = boto.utils.merge_meta(headers, metadata) - else: - headers[provider.metadata_directive_header] = 'COPY' - response = self.connection.make_request('PUT', self.name, new_key_name, - headers=headers) - body = response.read() - if response.status == 200: - key = self.new_key(new_key_name) - h = handler.XmlHandler(key, self) - xml.sax.parseString(body, h) - if hasattr(key, 'Error'): - raise provider.storage_copy_error(key.Code, key.Message, body) - key.handle_version_headers(response) - if preserve_acl: - self.set_xml_acl(acl, new_key_name) - return key - else: - raise provider.storage_response_error(response.status, response.reason, body) - - def set_canned_acl(self, acl_str, key_name='', headers=None, - version_id=None): - assert acl_str in CannedACLStrings - - if headers: - headers[self.connection.provider.acl_header] = acl_str - else: - headers={self.connection.provider.acl_header: acl_str} - - query_args='acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('PUT', self.name, key_name, - headers=headers, query_args=query_args) - body = response.read() - if response.status != 200: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_xml_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - if response.status != 200: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - return body - - def set_xml_acl(self, acl_str, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('PUT', self.name, key_name, - data=acl_str, - query_args=query_args, - headers=headers) - body = response.read() - if response.status != 200: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): - if isinstance(acl_or_str, Policy): - self.set_xml_acl(acl_or_str.to_xml(), key_name, - headers, version_id) - else: - self.set_canned_acl(acl_or_str, key_name, - headers, version_id) - - def get_acl(self, key_name='', headers=None, version_id=None): - query_args = 'acl' - if version_id: - query_args += '&versionId=%s' % version_id - response = self.connection.make_request('GET', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - if response.status == 200: - policy = Policy(self) - h = handler.XmlHandler(policy, self) - xml.sax.parseString(body, h) - return policy - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def make_public(self, recursive=False, headers=None): - self.set_canned_acl('public-read', headers=headers) - if recursive: - for key in self: - self.set_canned_acl('public-read', key.name, headers=headers) - - def add_email_grant(self, permission, email_address, - recursive=False, headers=None): - """ - Convenience method that provides a quick way to add an email grant - to a bucket. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL - and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type email_address: string - :param email_address: The email address associated with the AWS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in S3Permissions: - raise self.connection.provider.storage_permissions_error( - 'Unknown Permission: %s' % permission) - policy = self.get_acl(headers=headers) - policy.acl.add_email_grant(permission, email_address) - self.set_acl(policy, headers=headers) - if recursive: - for key in self: - key.add_email_grant(permission, email_address, headers=headers) - - def add_user_grant(self, permission, user_id, - recursive=False, headers=None): - """ - Convenience method that provides a quick way to add a canonical - user grant to a bucket. This method retrieves the current ACL, - creates a new grant based on the parameters passed in, adds that - grant to the ACL and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type user_id: string - :param user_id: The canonical user id associated with the AWS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - if permission not in S3Permissions: - raise self.connection.provider.storage_permissions_error( - 'Unknown Permission: %s' % permission) - policy = self.get_acl(headers=headers) - policy.acl.add_user_grant(permission, user_id) - self.set_acl(policy, headers=headers) - if recursive: - for key in self: - key.add_user_grant(permission, user_id, headers=headers) - - def list_grants(self, headers=None): - policy = self.get_acl(headers=headers) - return policy.acl.grants - - def get_location(self): - """ - Returns the LocationConstraint for the bucket. - - :rtype: str - :return: The LocationConstraint for the bucket or the empty - string if no constraint was specified when bucket - was created. - """ - response = self.connection.make_request('GET', self.name, - query_args='location') - body = response.read() - if response.status == 200: - rs = ResultSet(self) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs.LocationConstraint - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def enable_logging(self, target_bucket, target_prefix='', headers=None): - if isinstance(target_bucket, Bucket): - target_bucket = target_bucket.name - body = self.BucketLoggingBody % (target_bucket, target_prefix) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def disable_logging(self, headers=None): - body = self.EmptyBucketLoggingBody - response = self.connection.make_request('PUT', self.name, data=body, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_logging_status(self, headers=None): - response = self.connection.make_request('GET', self.name, - query_args='logging', headers=headers) - body = response.read() - if response.status == 200: - return body - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def set_as_logging_target(self, headers=None): - policy = self.get_acl(headers=headers) - g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) - g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) - policy.acl.add_grant(g1) - policy.acl.add_grant(g2) - self.set_acl(policy, headers=headers) - - def get_request_payment(self, headers=None): - response = self.connection.make_request('GET', self.name, - query_args='requestPayment', headers=headers) - body = response.read() - if response.status == 200: - return body - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def set_request_payment(self, payer='BucketOwner', headers=None): - body = self.BucketPaymentBody % payer - response = self.connection.make_request('PUT', self.name, data=body, - query_args='requestPayment', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def configure_versioning(self, versioning, mfa_delete=False, - mfa_token=None, headers=None): - """ - Configure versioning for this bucket. - Note: This feature is currently in beta release and is available - only in the Northern California region. - - :type versioning: bool - :param versioning: A boolean indicating whether version is - enabled (True) or disabled (False). - - :type mfa_delete: bool - :param mfa_delete: A boolean indicating whether the Multi-Factor - Authentication Delete feature is enabled (True) - or disabled (False). If mfa_delete is enabled - then all Delete operations will require the - token from your MFA device to be passed in - the request. - - :type mfa_token: tuple or list of strings - :param mfa_token: A tuple or list consisting of the serial number - from the MFA device and the current value of - the six-digit token associated with the device. - This value is required when you are changing - the status of the MfaDelete property of - the bucket. - """ - if versioning: - ver = 'Enabled' - else: - ver = 'Suspended' - if mfa_delete: - mfa = 'Enabled' - else: - mfa = 'Disabled' - body = self.VersioningBody % (ver, mfa) - if mfa_token: - if not headers: - headers = {} - provider = self.connection.provider - headers[provider.mfa_header] = ' '.join(mfa_token) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='versioning', headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_versioning_status(self, headers=None): - """ - Returns the current status of versioning on the bucket. - - :rtype: dict - :returns: A dictionary containing a key named 'Versioning' - that can have a value of either Enabled, Disabled, - or Suspended. Also, if MFADelete has ever been enabled - on the bucket, the dictionary will contain a key - named 'MFADelete' which will have a value of either - Enabled or Suspended. - """ - response = self.connection.make_request('GET', self.name, - query_args='versioning', headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 200: - d = {} - ver = re.search(self.VersionRE, body) - if ver: - d['Versioning'] = ver.group(1) - mfa = re.search(self.MFADeleteRE, body) - if mfa: - d['MfaDelete'] = mfa.group(1) - return d - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def configure_website(self, suffix, error_key='', headers=None): - """ - Configure this bucket to act as a website - - :type suffix: str - :param suffix: Suffix that is appended to a request that is for a - "directory" on the website endpoint (e.g. if the suffix - is index.html and you make a request to - samplebucket/images/ the data that is returned will - be for the object with the key name images/index.html). - The suffix must not be empty and must not include a - slash character. - - :type error_key: str - :param error_key: The object key name to use when a 4XX class - error occurs. This is optional. - - """ - if error_key: - error_frag = self.WebsiteErrorFragment % error_key - else: - error_frag = '' - body = self.WebsiteBody % (suffix, error_frag) - response = self.connection.make_request('PUT', self.name, data=body, - query_args='website', - headers=headers) - body = response.read() - if response.status == 200: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_website_configuration(self, headers=None): - """ - Returns the current status of website configuration on the bucket. - - :rtype: dict - :returns: A dictionary containing a Python representation - of the XML response from S3. The overall structure is: - - * WebsiteConfiguration - * IndexDocument - * Suffix : suffix that is appended to request that - is for a "directory" on the website endpoint - * ErrorDocument - * Key : name of object to serve when an error occurs - """ - response = self.connection.make_request('GET', self.name, - query_args='website', headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 200: - e = boto.jsonresponse.Element() - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def delete_website_configuration(self, headers=None): - """ - Removes all website configuration from the bucket. - """ - response = self.connection.make_request('DELETE', self.name, - query_args='website', headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 204: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def get_website_endpoint(self): - """ - Returns the fully qualified hostname to use is you want to access this - bucket as a website. This doesn't validate whether the bucket has - been correctly configured as a website or not. - """ - l = [self.name] - l.append(S3WebsiteEndpointTranslate.translate_region(self.get_location())) - l.append('.'.join(self.connection.host.split('.')[-2:])) - return '.'.join(l) - - def get_policy(self, headers=None): - response = self.connection.make_request('GET', self.name, - query_args='policy', headers=headers) - body = response.read() - if response.status == 200: - return body - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def set_policy(self, policy, headers=None): - response = self.connection.make_request('PUT', self.name, - data=policy, - query_args='policy', - headers=headers) - body = response.read() - if response.status >= 200 and response.status <= 204: - return True - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def initiate_multipart_upload(self, key_name, headers=None, reduced_redundancy=False): - """ - Start a multipart upload operation. - - :type key_name: string - :param key_name: The name of the key that will ultimately result from - this multipart upload operation. This will be exactly - as the key appears in the bucket after the upload - process has been completed. - - :type headers: dict - :param headers: Additional HTTP headers to send and store with the - resulting key in S3. - - :type reduced_redundancy: boolean - :param reduced_redundancy: In multipart uploads, the storage class is - specified when initiating the upload, - not when uploading individual parts. So - if you want the resulting key to use the - reduced redundancy storage class set this - flag when you initiate the upload. - """ - query_args = 'uploads' - if headers is None: - headers = {} - if reduced_redundancy: - storage_class_header = self.connection.provider.storage_class_header - if storage_class_header: - headers[storage_class_header] = 'REDUCED_REDUNDANCY' - # TODO: what if the provider doesn't support reduced redundancy? - # (see boto.s3.key.Key.set_contents_from_file) - response = self.connection.make_request('POST', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - boto.log.debug(body) - if response.status == 200: - resp = MultiPartUpload(self) - h = handler.XmlHandler(resp, self) - xml.sax.parseString(body, h) - return resp - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def complete_multipart_upload(self, key_name, upload_id, - xml_body, headers=None): - """ - Complete a multipart upload operation. - """ - query_args = 'uploadId=%s' % upload_id - if headers is None: - headers = {} - headers['Content-Type'] = 'text/xml' - response = self.connection.make_request('POST', self.name, key_name, - query_args=query_args, - headers=headers, data=xml_body) - contains_error = False - body = response.read() - # Some errors will be reported in the body of the response - # even though the HTTP response code is 200. This check - # does a quick and dirty peek in the body for an error element. - if body.find('') > 0: - contains_error = True - boto.log.debug(body) - if response.status == 200 and not contains_error: - resp = CompleteMultiPartUpload(self) - h = handler.XmlHandler(resp, self) - xml.sax.parseString(body, h) - return resp - else: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def cancel_multipart_upload(self, key_name, upload_id, headers=None): - query_args = 'uploadId=%s' % upload_id - response = self.connection.make_request('DELETE', self.name, key_name, - query_args=query_args, - headers=headers) - body = response.read() - boto.log.debug(body) - if response.status != 204: - raise self.connection.provider.storage_response_error( - response.status, response.reason, body) - - def delete(self, headers=None): - return self.connection.delete_bucket(self.name, headers=headers) - diff --git a/backup/src/boto/s3/bucketlistresultset.py b/backup/src/boto/s3/bucketlistresultset.py deleted file mode 100644 index 0123663..0000000 --- a/backup/src/boto/s3/bucketlistresultset.py +++ /dev/null @@ -1,139 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None): - """ - A generator function for listing keys in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_keys(prefix=prefix, marker=marker, - delimiter=delimiter, headers=headers) - for k in rs: - yield k - if k: - marker = k.name - more_results= rs.is_truncated - -class BucketListResultSet: - """ - A resultset for listing keys within a bucket. Uses the bucket_lister - generator function and implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of keys within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - - def __init__(self, bucket=None, prefix='', delimiter='', marker='', headers=None): - self.bucket = bucket - self.prefix = prefix - self.delimiter = delimiter - self.marker = marker - self.headers = headers - - def __iter__(self): - return bucket_lister(self.bucket, prefix=self.prefix, - delimiter=self.delimiter, marker=self.marker, headers=self.headers) - -def versioned_bucket_lister(bucket, prefix='', delimiter='', - key_marker='', version_id_marker='', headers=None): - """ - A generator function for listing versions in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_versions(prefix=prefix, key_marker=key_marker, - version_id_marker=version_id_marker, - delimiter=delimiter, headers=headers) - for k in rs: - yield k - key_marker = rs.next_key_marker - version_id_marker = rs.next_version_id_marker - more_results= rs.is_truncated - -class VersionedBucketListResultSet: - """ - A resultset for listing versions within a bucket. Uses the bucket_lister - generator function and implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of keys within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - - def __init__(self, bucket=None, prefix='', delimiter='', key_marker='', - version_id_marker='', headers=None): - self.bucket = bucket - self.prefix = prefix - self.delimiter = delimiter - self.key_marker = key_marker - self.version_id_marker = version_id_marker - self.headers = headers - - def __iter__(self): - return versioned_bucket_lister(self.bucket, prefix=self.prefix, - delimiter=self.delimiter, - key_marker=self.key_marker, - version_id_marker=self.version_id_marker, - headers=self.headers) - -def multipart_upload_lister(bucket, key_marker='', - upload_id_marker='', - headers=None): - """ - A generator function for listing multipart uploads in a bucket. - """ - more_results = True - k = None - while more_results: - rs = bucket.get_all_multipart_uploads(key_marker=key_marker, - upload_id_marker=upload_id_marker, - headers=headers) - for k in rs: - yield k - key_marker = rs.next_key_marker - upload_id_marker = rs.next_upload_id_marker - more_results= rs.is_truncated - -class MultiPartUploadListResultSet: - """ - A resultset for listing multipart uploads within a bucket. - Uses the multipart_upload_lister generator function and - implements the iterator interface. This - transparently handles the results paging from S3 so even if you have - many thousands of uploads within the bucket you can iterate over all - keys in a reasonably efficient manner. - """ - def __init__(self, bucket=None, key_marker='', - upload_id_marker='', headers=None): - self.bucket = bucket - self.key_marker = key_marker - self.upload_id_marker = upload_id_marker - self.headers = headers - - def __iter__(self): - return multipart_upload_lister(self.bucket, - key_marker=self.key_marker, - upload_id_marker=self.upload_id_marker, - headers=self.headers) - - diff --git a/backup/src/boto/s3/connection.py b/backup/src/boto/s3/connection.py deleted file mode 100644 index 25ba4ab..0000000 --- a/backup/src/boto/s3/connection.py +++ /dev/null @@ -1,401 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import xml.sax -import urllib, base64 -import time -import boto.utils -from boto.connection import AWSAuthConnection -from boto import handler -from boto.provider import Provider -from boto.s3.bucket import Bucket -from boto.s3.key import Key -from boto.resultset import ResultSet -from boto.exception import BotoClientError - -def check_lowercase_bucketname(n): - """ - Bucket names must not contain uppercase characters. We check for - this by appending a lowercase character and testing with islower(). - Note this also covers cases like numeric bucket names with dashes. - - >>> check_lowercase_bucketname("Aaaa") - Traceback (most recent call last): - ... - BotoClientError: S3Error: Bucket names cannot contain upper-case - characters when using either the sub-domain or virtual hosting calling - format. - - >>> check_lowercase_bucketname("1234-5678-9123") - True - >>> check_lowercase_bucketname("abcdefg1234") - True - """ - if not (n + 'a').islower(): - raise BotoClientError("Bucket names cannot contain upper-case " \ - "characters when using either the sub-domain or virtual " \ - "hosting calling format.") - return True - -def assert_case_insensitive(f): - def wrapper(*args, **kwargs): - if len(args) == 3 and check_lowercase_bucketname(args[2]): - pass - return f(*args, **kwargs) - return wrapper - -class _CallingFormat: - - def build_url_base(self, connection, protocol, server, bucket, key=''): - url_base = '%s://' % protocol - url_base += self.build_host(server, bucket) - url_base += connection.get_path(self.build_path_base(bucket, key)) - return url_base - - def build_host(self, server, bucket): - if bucket == '': - return server - else: - return self.get_bucket_server(server, bucket) - - def build_auth_path(self, bucket, key=''): - path = '' - if bucket != '': - path = '/' + bucket - return path + '/%s' % urllib.quote(key) - - def build_path_base(self, bucket, key=''): - return '/%s' % urllib.quote(key) - -class SubdomainCallingFormat(_CallingFormat): - - @assert_case_insensitive - def get_bucket_server(self, server, bucket): - return '%s.%s' % (bucket, server) - -class VHostCallingFormat(_CallingFormat): - - @assert_case_insensitive - def get_bucket_server(self, server, bucket): - return bucket - -class OrdinaryCallingFormat(_CallingFormat): - - def get_bucket_server(self, server, bucket): - return server - - def build_path_base(self, bucket, key=''): - path_base = '/' - if bucket: - path_base += "%s/" % bucket - return path_base + urllib.quote(key) - -class Location: - DEFAULT = '' # US Classic Region - EU = 'EU' - USWest = 'us-west-1' - APSoutheast = 'ap-southeast-1' - -class S3Connection(AWSAuthConnection): - - DefaultHost = 's3.amazonaws.com' - QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, - host=DefaultHost, debug=0, https_connection_factory=None, - calling_format=SubdomainCallingFormat(), path='/', provider='aws', - bucket_class=Bucket): - self.calling_format = calling_format - self.bucket_class = bucket_class - AWSAuthConnection.__init__(self, host, - aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - debug=debug, https_connection_factory=https_connection_factory, - path=path, provider=provider) - - def _required_auth_capability(self): - return ['s3'] - - def __iter__(self): - for bucket in self.get_all_buckets(): - yield bucket - - def __contains__(self, bucket_name): - return not (self.lookup(bucket_name) is None) - - def set_bucket_class(self, bucket_class): - """ - Set the Bucket class associated with this bucket. By default, this - would be the boto.s3.key.Bucket class but if you want to subclass that - for some reason this allows you to associate your new class. - - :type bucket_class: class - :param bucket_class: A subclass of Bucket that can be more specific - """ - self.bucket_class = bucket_class - - def build_post_policy(self, expiration_time, conditions): - """ - Taken from the AWS book Python examples and modified for use with boto - """ - assert type(expiration_time) == time.struct_time, \ - 'Policy document must include a valid expiration Time object' - - # Convert conditions object mappings to condition statements - - return '{"expiration": "%s",\n"conditions": [%s]}' % \ - (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions)) - - - def build_post_form_args(self, bucket_name, key, expires_in = 6000, - acl = None, success_action_redirect = None, max_content_length = None, - http_method = "http", fields=None, conditions=None): - """ - Taken from the AWS book Python examples and modified for use with boto - This only returns the arguments required for the post form, not the actual form - This does not return the file input field which also needs to be added - - :param bucket_name: Bucket to submit to - :type bucket_name: string - - :param key: Key name, optionally add ${filename} to the end to attach the submitted filename - :type key: string - - :param expires_in: Time (in seconds) before this expires, defaults to 6000 - :type expires_in: integer - - :param acl: ACL rule to use, if any - :type acl: :class:`boto.s3.acl.ACL` - - :param success_action_redirect: URL to redirect to on success - :type success_action_redirect: string - - :param max_content_length: Maximum size for this file - :type max_content_length: integer - - :type http_method: string - :param http_method: HTTP Method to use, "http" or "https" - - - :rtype: dict - :return: A dictionary containing field names/values as well as a url to POST to - - .. code-block:: python - - { - "action": action_url_to_post_to, - "fields": [ - { - "name": field_name, - "value": field_value - }, - { - "name": field_name2, - "value": field_value2 - } - ] - } - - """ - if fields == None: - fields = [] - if conditions == None: - conditions = [] - expiration = time.gmtime(int(time.time() + expires_in)) - - # Generate policy document - conditions.append('{"bucket": "%s"}' % bucket_name) - if key.endswith("${filename}"): - conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")]) - else: - conditions.append('{"key": "%s"}' % key) - if acl: - conditions.append('{"acl": "%s"}' % acl) - fields.append({ "name": "acl", "value": acl}) - if success_action_redirect: - conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect) - fields.append({ "name": "success_action_redirect", "value": success_action_redirect}) - if max_content_length: - conditions.append('["content-length-range", 0, %i]' % max_content_length) - fields.append({"name":'content-length-range', "value": "0,%i" % max_content_length}) - - policy = self.build_post_policy(expiration, conditions) - - # Add the base64-encoded policy document as the 'policy' field - policy_b64 = base64.b64encode(policy) - fields.append({"name": "policy", "value": policy_b64}) - - # Add the AWS access key as the 'AWSAccessKeyId' field - fields.append({"name": "AWSAccessKeyId", "value": self.aws_access_key_id}) - - # Add signature for encoded policy document as the 'AWSAccessKeyId' field - signature = self._auth_handler.sign_string(policy_b64) - fields.append({"name": "signature", "value": signature}) - fields.append({"name": "key", "value": key}) - - # HTTPS protocol will be used if the secure HTTP option is enabled. - url = '%s://%s/' % (http_method, self.calling_format.build_host(self.server_name(), bucket_name)) - - return {"action": url, "fields": fields} - - - def generate_url(self, expires_in, method, bucket='', key='', - headers=None, query_auth=True, force_http=False): - if not headers: - headers = {} - expires = int(time.time() + expires_in) - auth_path = self.calling_format.build_auth_path(bucket, key) - auth_path = self.get_path(auth_path) - c_string = boto.utils.canonical_string(method, auth_path, headers, - expires, self.provider) - b64_hmac = self._auth_handler.sign_string(c_string) - encoded_canonical = urllib.quote_plus(b64_hmac) - self.calling_format.build_path_base(bucket, key) - if query_auth: - query_part = '?' + self.QueryString % (encoded_canonical, expires, - self.aws_access_key_id) - sec_hdr = self.provider.security_token_header - if sec_hdr in headers: - query_part += ('&%s=%s' % (sec_hdr, - urllib.quote(headers[sec_hdr]))); - else: - query_part = '' - if force_http: - protocol = 'http' - port = 80 - else: - protocol = self.protocol - port = self.port - return self.calling_format.build_url_base(self, protocol, self.server_name(port), - bucket, key) + query_part - - def get_all_buckets(self, headers=None): - response = self.make_request('GET', headers=headers) - body = response.read() - if response.status > 300: - raise self.provider.storage_response_error( - response.status, response.reason, body) - rs = ResultSet([('Bucket', self.bucket_class)]) - h = handler.XmlHandler(rs, self) - xml.sax.parseString(body, h) - return rs - - def get_canonical_user_id(self, headers=None): - """ - Convenience method that returns the "CanonicalUserID" of the user who's credentials - are associated with the connection. The only way to get this value is to do a GET - request on the service which returns all buckets associated with the account. As part - of that response, the canonical userid is returned. This method simply does all of - that and then returns just the user id. - - :rtype: string - :return: A string containing the canonical user id. - """ - rs = self.get_all_buckets(headers=headers) - return rs.ID - - def get_bucket(self, bucket_name, validate=True, headers=None): - bucket = self.bucket_class(self, bucket_name) - if validate: - bucket.get_all_keys(headers, maxkeys=0) - return bucket - - def lookup(self, bucket_name, validate=True, headers=None): - try: - bucket = self.get_bucket(bucket_name, validate, headers=headers) - except: - bucket = None - return bucket - - def create_bucket(self, bucket_name, headers=None, - location=Location.DEFAULT, policy=None): - """ - Creates a new located bucket. By default it's in the USA. You can pass - Location.EU to create an European bucket. - - :type bucket_name: string - :param bucket_name: The name of the new bucket - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type location: :class:`boto.s3.connection.Location` - :param location: The location of the new bucket - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - """ - check_lowercase_bucketname(bucket_name) - - if policy: - if headers: - headers[self.provider.acl_header] = policy - else: - headers = {self.provider.acl_header : policy} - if location == Location.DEFAULT: - data = '' - else: - data = '' + \ - location + '' - response = self.make_request('PUT', bucket_name, headers=headers, - data=data) - body = response.read() - if response.status == 409: - raise self.provider.storage_create_error( - response.status, response.reason, body) - if response.status == 200: - return self.bucket_class(self, bucket_name) - else: - raise self.provider.storage_response_error( - response.status, response.reason, body) - - def delete_bucket(self, bucket, headers=None): - response = self.make_request('DELETE', bucket, headers=headers) - body = response.read() - if response.status != 204: - raise self.provider.storage_response_error( - response.status, response.reason, body) - - def make_request(self, method, bucket='', key='', headers=None, data='', - query_args=None, sender=None, override_num_retries=None): - if isinstance(bucket, self.bucket_class): - bucket = bucket.name - if isinstance(key, Key): - key = key.name - path = self.calling_format.build_path_base(bucket, key) - boto.log.debug('path=%s' % path) - auth_path = self.calling_format.build_auth_path(bucket, key) - boto.log.debug('auth_path=%s' % auth_path) - host = self.calling_format.build_host(self.server_name(), bucket) - if query_args: - path += '?' + query_args - boto.log.debug('path=%s' % path) - auth_path += '?' + query_args - boto.log.debug('auth_path=%s' % auth_path) - return AWSAuthConnection.make_request(self, method, path, headers, - data, host, auth_path, sender, - override_num_retries=override_num_retries) - diff --git a/backup/src/boto/s3/deletemarker.py b/backup/src/boto/s3/deletemarker.py deleted file mode 100644 index 3462d42..0000000 --- a/backup/src/boto/s3/deletemarker.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.s3.user import User - -class DeleteMarker: - def __init__(self, bucket=None, name=None): - self.bucket = bucket - self.name = name - self.is_latest = False - self.last_modified = None - self.owner = None - - def startElement(self, name, attrs, connection): - if name == 'Owner': - self.owner = User(self) - return self.owner - else: - return None - - def endElement(self, name, value, connection): - if name == 'Key': - self.name = value.encode('utf-8') - elif name == 'IsLatest': - if value == 'true': - self.is_lastest = True - else: - self.is_latest = False - elif name == 'LastModified': - self.last_modified = value - elif name == 'Owner': - pass - elif name == 'VersionId': - self.version_id = value - else: - setattr(self, name, value) - - diff --git a/backup/src/boto/s3/key.py b/backup/src/boto/s3/key.py deleted file mode 100644 index c7e77f4..0000000 --- a/backup/src/boto/s3/key.py +++ /dev/null @@ -1,1059 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import mimetypes -import os -import rfc822 -import StringIO -import base64 -import boto.utils -from boto.exception import BotoClientError -from boto.provider import Provider -from boto.s3.user import User -from boto import UserAgent -try: - from hashlib import md5 -except ImportError: - from md5 import md5 - - -class Key(object): - - DefaultContentType = 'application/octet-stream' - - BufferSize = 8192 - - def __init__(self, bucket=None, name=None): - self.bucket = bucket - self.name = name - self.metadata = {} - self.cache_control = None - self.content_type = self.DefaultContentType - self.content_encoding = None - self.filename = None - self.etag = None - self.last_modified = None - self.owner = None - self.storage_class = 'STANDARD' - self.md5 = None - self.base64md5 = None - self.path = None - self.resp = None - self.mode = None - self.size = None - self.version_id = None - self.source_version_id = None - self.delete_marker = False - - def __repr__(self): - if self.bucket: - return '' % (self.bucket.name, self.name) - else: - return '' % self.name - - def __getattr__(self, name): - if name == 'key': - return self.name - else: - raise AttributeError - - def __setattr__(self, name, value): - if name == 'key': - self.__dict__['name'] = value - else: - self.__dict__[name] = value - - def __iter__(self): - return self - - @property - def provider(self): - provider = None - if self.bucket: - if self.bucket.connection: - provider = self.bucket.connection.provider - return provider - - def get_md5_from_hexdigest(self, md5_hexdigest): - """ - A utility function to create the 2-tuple (md5hexdigest, base64md5) - from just having a precalculated md5_hexdigest. - """ - import binascii - digest = binascii.unhexlify(md5_hexdigest) - base64md5 = base64.encodestring(digest) - if base64md5[-1] == '\n': - base64md5 = base64md5[0:-1] - return (md5_hexdigest, base64md5) - - def handle_version_headers(self, resp, force=False): - provider = self.bucket.connection.provider - # If the Key object already has a version_id attribute value, it - # means that it represents an explicit version and the user is - # doing a get_contents_*(version_id=) to retrieve another - # version of the Key. In that case, we don't really want to - # overwrite the version_id in this Key object. Comprende? - if self.version_id is None or force: - self.version_id = resp.getheader(provider.version_id, None) - self.source_version_id = resp.getheader(provider.copy_source_version_id, None) - if resp.getheader(provider.delete_marker, 'false') == 'true': - self.delete_marker = True - else: - self.delete_marker = False - - def open_read(self, headers=None, query_args=None, - override_num_retries=None, response_headers=None): - """ - Open this key for reading - - :type headers: dict - :param headers: Headers to pass in the web request - - :type query_args: string - :param query_args: Arguments to pass in the query string (ie, 'torrent') - - :type override_num_retries: int - :param override_num_retries: If not None will override configured - num_retries parameter for underlying GET. - - :type response_headers: dict - :param response_headers: A dictionary containing HTTP headers/values - that will override any headers associated with - the stored object in the response. - See http://goo.gl/EWOPb for details. - """ - if self.resp == None: - self.mode = 'r' - - provider = self.bucket.connection.provider - self.resp = self.bucket.connection.make_request( - 'GET', self.bucket.name, self.name, headers, - query_args=query_args, - override_num_retries=override_num_retries) - if self.resp.status < 199 or self.resp.status > 299: - body = self.resp.read() - raise provider.storage_response_error(self.resp.status, - self.resp.reason, body) - response_headers = self.resp.msg - self.metadata = boto.utils.get_aws_metadata(response_headers, - provider) - for name,value in response_headers.items(): - if name.lower() == 'content-length': - self.size = int(value) - elif name.lower() == 'etag': - self.etag = value - elif name.lower() == 'content-type': - self.content_type = value - elif name.lower() == 'content-encoding': - self.content_encoding = value - elif name.lower() == 'last-modified': - self.last_modified = value - elif name.lower() == 'cache-control': - self.cache_control = value - self.handle_version_headers(self.resp) - - def open_write(self, headers=None, override_num_retries=None): - """ - Open this key for writing. - Not yet implemented - - :type headers: dict - :param headers: Headers to pass in the write request - - :type override_num_retries: int - :param override_num_retries: If not None will override configured - num_retries parameter for underlying PUT. - """ - raise BotoClientError('Not Implemented') - - def open(self, mode='r', headers=None, query_args=None, - override_num_retries=None): - if mode == 'r': - self.mode = 'r' - self.open_read(headers=headers, query_args=query_args, - override_num_retries=override_num_retries) - elif mode == 'w': - self.mode = 'w' - self.open_write(headers=headers, - override_num_retries=override_num_retries) - else: - raise BotoClientError('Invalid mode: %s' % mode) - - closed = False - def close(self): - if self.resp: - self.resp.read() - self.resp = None - self.mode = None - self.closed = True - - def next(self): - """ - By providing a next method, the key object supports use as an iterator. - For example, you can now say: - - for bytes in key: - write bytes to a file or whatever - - All of the HTTP connection stuff is handled for you. - """ - self.open_read() - data = self.resp.read(self.BufferSize) - if not data: - self.close() - raise StopIteration - return data - - def read(self, size=0): - if size == 0: - size = self.BufferSize - self.open_read() - data = self.resp.read(size) - if not data: - self.close() - return data - - def change_storage_class(self, new_storage_class, dst_bucket=None): - """ - Change the storage class of an existing key. - Depending on whether a different destination bucket is supplied - or not, this will either move the item within the bucket, preserving - all metadata and ACL info bucket changing the storage class or it - will copy the item to the provided destination bucket, also - preserving metadata and ACL info. - - :type new_storage_class: string - :param new_storage_class: The new storage class for the Key. - Possible values are: - * STANDARD - * REDUCED_REDUNDANCY - - :type dst_bucket: string - :param dst_bucket: The name of a destination bucket. If not - provided the current bucket of the key - will be used. - - """ - if new_storage_class == 'STANDARD': - return self.copy(self.bucket.name, self.name, - reduced_redundancy=False, preserve_acl=True) - elif new_storage_class == 'REDUCED_REDUNDANCY': - return self.copy(self.bucket.name, self.name, - reduced_redundancy=True, preserve_acl=True) - else: - raise BotoClientError('Invalid storage class: %s' % - new_storage_class) - - def copy(self, dst_bucket, dst_key, metadata=None, - reduced_redundancy=False, preserve_acl=False): - """ - Copy this Key to another bucket. - - :type dst_bucket: string - :param dst_bucket: The name of the destination bucket - - :type dst_key: string - :param dst_key: The name of the destination key - - :type metadata: dict - :param metadata: Metadata to be associated with new key. - If metadata is supplied, it will replace the - metadata of the source key being copied. - If no metadata is supplied, the source key's - metadata will be copied to the new key. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will force the storage - class of the new Key to be - REDUCED_REDUNDANCY regardless of the - storage class of the key being copied. - The Reduced Redundancy Storage (RRS) - feature of S3, provides lower - redundancy at lower storage cost. - - :type preserve_acl: bool - :param preserve_acl: If True, the ACL from the source key - will be copied to the destination - key. If False, the destination key - will have the default ACL. - Note that preserving the ACL in the - new key object will require two - additional API calls to S3, one to - retrieve the current ACL and one to - set that ACL on the new object. If - you don't care about the ACL, a value - of False will be significantly more - efficient. - - :rtype: :class:`boto.s3.key.Key` or subclass - :returns: An instance of the newly created key object - """ - dst_bucket = self.bucket.connection.lookup(dst_bucket) - if reduced_redundancy: - storage_class = 'REDUCED_REDUNDANCY' - else: - storage_class = self.storage_class - return dst_bucket.copy_key(dst_key, self.bucket.name, - self.name, metadata, - storage_class=storage_class, - preserve_acl=preserve_acl) - - def startElement(self, name, attrs, connection): - if name == 'Owner': - self.owner = User(self) - return self.owner - else: - return None - - def endElement(self, name, value, connection): - if name == 'Key': - self.name = value.encode('utf-8') - elif name == 'ETag': - self.etag = value - elif name == 'LastModified': - self.last_modified = value - elif name == 'Size': - self.size = int(value) - elif name == 'StorageClass': - self.storage_class = value - elif name == 'Owner': - pass - elif name == 'VersionId': - self.version_id = value - else: - setattr(self, name, value) - - def exists(self): - """ - Returns True if the key exists - - :rtype: bool - :return: Whether the key exists on S3 - """ - return bool(self.bucket.lookup(self.name)) - - def delete(self): - """ - Delete this key from S3 - """ - return self.bucket.delete_key(self.name, version_id=self.version_id) - - def get_metadata(self, name): - return self.metadata.get(name) - - def set_metadata(self, name, value): - self.metadata[name] = value - - def update_metadata(self, d): - self.metadata.update(d) - - # convenience methods for setting/getting ACL - def set_acl(self, acl_str, headers=None): - if self.bucket != None: - self.bucket.set_acl(acl_str, self.name, headers=headers) - - def get_acl(self, headers=None): - if self.bucket != None: - return self.bucket.get_acl(self.name, headers=headers) - - def get_xml_acl(self, headers=None): - if self.bucket != None: - return self.bucket.get_xml_acl(self.name, headers=headers) - - def set_xml_acl(self, acl_str, headers=None): - if self.bucket != None: - return self.bucket.set_xml_acl(acl_str, self.name, headers=headers) - - def set_canned_acl(self, acl_str, headers=None): - return self.bucket.set_canned_acl(acl_str, self.name, headers) - - def make_public(self, headers=None): - return self.bucket.set_canned_acl('public-read', self.name, headers) - - def generate_url(self, expires_in, method='GET', headers=None, - query_auth=True, force_http=False): - """ - Generate a URL to access this key. - - :type expires_in: int - :param expires_in: How long the url is valid for, in seconds - - :type method: string - :param method: The method to use for retrieving the file (default is GET) - - :type headers: dict - :param headers: Any headers to pass along in the request - - :type query_auth: bool - :param query_auth: - - :rtype: string - :return: The URL to access the key - """ - return self.bucket.connection.generate_url(expires_in, method, - self.bucket.name, self.name, - headers, query_auth, force_http) - - def send_file(self, fp, headers=None, cb=None, num_cb=10, query_args=None): - """ - Upload a file to a key into a bucket on S3. - - :type fp: file - :param fp: The file pointer to upload - - :type headers: dict - :param headers: The headers to pass along with the PUT request - - :type cb: function - :param cb: a callback function that will be called to report - progress on the upload. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted to S3 and the second representing - the total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter this parameter determines the granularity - of the callback by defining the maximum number of - times the callback will be called during the file - transfer. Providing a negative integer will cause - your callback to be called with each buffer read. - - """ - provider = self.bucket.connection.provider - - def sender(http_conn, method, path, data, headers): - http_conn.putrequest(method, path) - for key in headers: - http_conn.putheader(key, headers[key]) - http_conn.endheaders() - fp.seek(0) - save_debug = self.bucket.connection.debug - self.bucket.connection.debug = 0 - http_conn.set_debuglevel(0) - if cb: - if num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb-2) - elif num_cb < 0: - cb_count = -1 - else: - cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) - l = fp.read(self.BufferSize) - while len(l) > 0: - http_conn.send(l) - if cb: - total_bytes += len(l) - i += 1 - if i == cb_count or cb_count == -1: - cb(total_bytes, self.size) - i = 0 - l = fp.read(self.BufferSize) - if cb: - cb(total_bytes, self.size) - response = http_conn.getresponse() - body = response.read() - fp.seek(0) - http_conn.set_debuglevel(save_debug) - self.bucket.connection.debug = save_debug - if response.status == 500 or response.status == 503 or \ - response.getheader('location'): - # we'll try again - return response - elif response.status >= 200 and response.status <= 299: - self.etag = response.getheader('etag') - if self.etag != '"%s"' % self.md5: - raise provider.storage_data_error( - 'ETag from S3 did not match computed MD5') - return response - else: - raise provider.storage_response_error( - response.status, response.reason, body) - - if not headers: - headers = {} - else: - headers = headers.copy() - headers['User-Agent'] = UserAgent - headers['Content-MD5'] = self.base64md5 - if self.storage_class != 'STANDARD': - headers[provider.storage_class_header] = self.storage_class - if headers.has_key('Content-Encoding'): - self.content_encoding = headers['Content-Encoding'] - if headers.has_key('Content-Type'): - self.content_type = headers['Content-Type'] - elif self.path: - self.content_type = mimetypes.guess_type(self.path)[0] - if self.content_type == None: - self.content_type = self.DefaultContentType - headers['Content-Type'] = self.content_type - else: - headers['Content-Type'] = self.content_type - headers['Content-Length'] = str(self.size) - headers['Expect'] = '100-Continue' - headers = boto.utils.merge_meta(headers, self.metadata, provider) - resp = self.bucket.connection.make_request('PUT', self.bucket.name, - self.name, headers, - sender=sender, - query_args=query_args) - self.handle_version_headers(resp, force=True) - - def compute_md5(self, fp): - """ - :type fp: file - :param fp: File pointer to the file to MD5 hash. The file pointer will be - reset to the beginning of the file before the method returns. - - :rtype: tuple - :return: A tuple containing the hex digest version of the MD5 hash - as the first element and the base64 encoded version of the - plain digest as the second element. - """ - m = md5() - fp.seek(0) - s = fp.read(self.BufferSize) - while s: - m.update(s) - s = fp.read(self.BufferSize) - hex_md5 = m.hexdigest() - base64md5 = base64.encodestring(m.digest()) - if base64md5[-1] == '\n': - base64md5 = base64md5[0:-1] - self.size = fp.tell() - fp.seek(0) - return (hex_md5, base64md5) - - def set_contents_from_file(self, fp, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False, query_args=None): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the contents of the file pointed to by 'fp' as the - contents. - - :type fp: file - :param fp: the file whose contents to upload - - :type headers: dict - :param headers: additional HTTP headers that will be sent with the PUT request. - - :type replace: bool - :param replace: If this parameter is False, the method - will first check to see if an object exists in the - bucket with the same key. If it does, it won't - overwrite it. The default value is True which will - overwrite the object. - - :type cb: function - :param cb: a callback function that will be called to report - progress on the upload. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted to S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - - """ - provider = self.bucket.connection.provider - if headers is None: - headers = {} - if policy: - headers[provider.acl_header] = policy - if reduced_redundancy: - self.storage_class = 'REDUCED_REDUNDANCY' - if provider.storage_class_header: - headers[provider.storage_class_header] = self.storage_class - # TODO - What if the provider doesn't support reduced reduncancy? - # What if different providers provide different classes? - if hasattr(fp, 'name'): - self.path = fp.name - if self.bucket != None: - if not md5: - md5 = self.compute_md5(fp) - else: - # even if md5 is provided, still need to set size of content - fp.seek(0, 2) - self.size = fp.tell() - fp.seek(0) - self.md5 = md5[0] - self.base64md5 = md5[1] - if self.name == None: - self.name = self.md5 - if not replace: - k = self.bucket.lookup(self.name) - if k: - return - self.send_file(fp, headers, cb, num_cb, query_args) - - def set_contents_from_filename(self, filename, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the contents of the file named by 'filename'. - See set_contents_from_file method for details about the - parameters. - - :type filename: string - :param filename: The name of the file that you want to put onto S3 - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type replace: bool - :param replace: If True, replaces the contents of the file if it already exists. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - """ - fp = open(filename, 'rb') - self.set_contents_from_file(fp, headers, replace, cb, num_cb, - policy, md5, reduced_redundancy) - fp.close() - - def set_contents_from_string(self, s, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - """ - Store an object in S3 using the name of the Key object as the - key in S3 and the string 's' as the contents. - See set_contents_from_file method for details about the - parameters. - - :type headers: dict - :param headers: Additional headers to pass along with the request to AWS. - - :type replace: bool - :param replace: If True, replaces the contents of the file if it already exists. - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type policy: :class:`boto.s3.acl.CannedACLStrings` - :param policy: A canned ACL policy that will be applied to the new key in S3. - - :type md5: A tuple containing the hexdigest version of the MD5 checksum of the - file as the first element and the Base64-encoded version of the plain - checksum as the second element. This is the same format returned by - the compute_md5 method. - :param md5: If you need to compute the MD5 for any reason prior to upload, - it's silly to have to do it twice so this param, if present, will be - used as the MD5 values of the file. Otherwise, the checksum will be computed. - - :type reduced_redundancy: bool - :param reduced_redundancy: If True, this will set the storage - class of the new Key to be - REDUCED_REDUNDANCY. The Reduced Redundancy - Storage (RRS) feature of S3, provides lower - redundancy at lower storage cost. - """ - fp = StringIO.StringIO(s) - r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, - policy, md5, reduced_redundancy) - fp.close() - return r - - def get_file(self, fp, headers=None, cb=None, num_cb=10, - torrent=False, version_id=None, override_num_retries=None, - response_headers=None): - """ - Retrieves a file from an S3 Key - - :type fp: file - :param fp: File pointer to put the data into - - :type headers: string - :param: headers to send when retrieving the files - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from S3 and the second representing - the total number of bytes that need to be transmitted. - - - :type cb: int - :param num_cb: (optional) If a callback is specified with the cb parameter - this parameter determines the granularity of the callback by defining - the maximum number of times the callback will be called during the file transfer. - - :type torrent: bool - :param torrent: Flag for whether to get a torrent for the file - - :type override_num_retries: int - :param override_num_retries: If not None will override configured - num_retries parameter for underlying GET. - - :type response_headers: dict - :param response_headers: A dictionary containing HTTP headers/values - that will override any headers associated with - the stored object in the response. - See http://goo.gl/EWOPb for details. - """ - if cb: - if num_cb > 2: - cb_count = self.size / self.BufferSize / (num_cb-2) - elif num_cb < 0: - cb_count = -1 - else: - cb_count = 0 - i = total_bytes = 0 - cb(total_bytes, self.size) - save_debug = self.bucket.connection.debug - if self.bucket.connection.debug == 1: - self.bucket.connection.debug = 0 - - query_args = [] - if torrent: - query_args.append('torrent') - # If a version_id is passed in, use that. If not, check to see - # if the Key object has an explicit version_id and, if so, use that. - # Otherwise, don't pass a version_id query param. - if version_id is None: - version_id = self.version_id - if version_id: - query_args.append('versionId=%s' % version_id) - if response_headers: - for key in response_headers: - query_args.append('%s=%s' % (key, response_headers[key])) - query_args = '&'.join(query_args) - self.open('r', headers, query_args=query_args, - override_num_retries=override_num_retries) - for bytes in self: - fp.write(bytes) - if cb: - total_bytes += len(bytes) - i += 1 - if i == cb_count or cb_count == -1: - cb(total_bytes, self.size) - i = 0 - if cb: - cb(total_bytes, self.size) - self.close() - self.bucket.connection.debug = save_debug - - def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): - """ - Get a torrent file (see to get_file) - - :type fp: file - :param fp: The file pointer of where to put the torrent - - :type headers: dict - :param headers: Headers to be passed - - :type cb: function - :param cb: (optional) a callback function that will be called to - report progress on the download. The callback should - accept two integer parameters, the first representing - the number of bytes that have been successfully - transmitted from S3 and the second representing the - total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the - cb parameter this parameter determines the - granularity of the callback by defining the - maximum number of times the callback will be - called during the file transfer. - - """ - return self.get_file(fp, headers, cb, num_cb, torrent=True) - - def get_contents_to_file(self, fp, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None, - res_download_handler=None, - response_headers=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Write the contents of the object to the file pointed - to by 'fp'. - - :type fp: File -like object - :param fp: - - :type headers: dict - :param headers: additional HTTP headers that will be sent with - the GET request. - - :type cb: function - :param cb: (optional) a callback function that will be called to - report progress on the download. The callback should - accept two integer parameters, the first representing - the number of bytes that have been successfully - transmitted from S3 and the second representing the - total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the - cb parameter this parameter determines the - granularity of the callback by defining the - maximum number of times the callback will be - called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent - file as a string. - - :type res_upload_handler: ResumableDownloadHandler - :param res_download_handler: If provided, this handler will - perform the download. - - :type response_headers: dict - :param response_headers: A dictionary containing HTTP headers/values - that will override any headers associated with - the stored object in the response. - See http://goo.gl/EWOPb for details. - """ - if self.bucket != None: - if res_download_handler: - res_download_handler.get_file(self, fp, headers, cb, num_cb, - torrent=torrent, - version_id=version_id) - else: - self.get_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id, - response_headers=response_headers) - - def get_contents_to_filename(self, filename, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None, - res_download_handler=None, - response_headers=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Store contents of the object to a file named by 'filename'. - See get_contents_to_file method for details about the - parameters. - - :type filename: string - :param filename: The filename of where to put the file contents - - :type headers: dict - :param headers: Any additional headers to send in the request - - :type cb: function - :param cb: (optional) a callback function that will be called to - report progress on the download. The callback should - accept two integer parameters, the first representing - the number of bytes that have been successfully - transmitted from S3 and the second representing the - total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the - cb parameter this parameter determines the - granularity of the callback by defining the - maximum number of times the callback will be - called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent file - as a string. - - :type res_upload_handler: ResumableDownloadHandler - :param res_download_handler: If provided, this handler will - perform the download. - - :type response_headers: dict - :param response_headers: A dictionary containing HTTP headers/values - that will override any headers associated with - the stored object in the response. - See http://goo.gl/EWOPb for details. - """ - fp = open(filename, 'wb') - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id, - res_download_handler=res_download_handler, - response_headers=response_headers) - fp.close() - # if last_modified date was sent from s3, try to set file's timestamp - if self.last_modified != None: - try: - modified_tuple = rfc822.parsedate_tz(self.last_modified) - modified_stamp = int(rfc822.mktime_tz(modified_tuple)) - os.utime(fp.name, (modified_stamp, modified_stamp)) - except Exception: pass - - def get_contents_as_string(self, headers=None, - cb=None, num_cb=10, - torrent=False, - version_id=None, - response_headers=None): - """ - Retrieve an object from S3 using the name of the Key object as the - key in S3. Return the contents of the object as a string. - See get_contents_to_file method for details about the - parameters. - - :type headers: dict - :param headers: Any additional headers to send in the request - - :type cb: function - :param cb: (optional) a callback function that will be called to - report progress on the download. The callback should - accept two integer parameters, the first representing - the number of bytes that have been successfully - transmitted from S3 and the second representing the - total number of bytes that need to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the - cb parameter this parameter determines the - granularity of the callback by defining the - maximum number of times the callback will be - called during the file transfer. - - :type torrent: bool - :param torrent: If True, returns the contents of a torrent file - as a string. - - :type response_headers: dict - :param response_headers: A dictionary containing HTTP headers/values - that will override any headers associated with - the stored object in the response. - See http://goo.gl/EWOPb for details. - - :rtype: string - :returns: The contents of the file as a string - """ - fp = StringIO.StringIO() - self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, - version_id=version_id, - response_headers=response_headers) - return fp.getvalue() - - def add_email_grant(self, permission, email_address, headers=None): - """ - Convenience method that provides a quick way to add an email grant - to a key. This method retrieves the current ACL, creates a new - grant based on the parameters passed in, adds that grant to the ACL - and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type email_address: string - :param email_address: The email address associated with the AWS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - policy = self.get_acl(headers=headers) - policy.acl.add_email_grant(permission, email_address) - self.set_acl(policy, headers=headers) - - def add_user_grant(self, permission, user_id, headers=None): - """ - Convenience method that provides a quick way to add a canonical - user grant to a key. This method retrieves the current ACL, - creates a new grant based on the parameters passed in, adds that - grant to the ACL and then PUT's the new ACL back to S3. - - :type permission: string - :param permission: The permission being granted. Should be one of: - (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). - - :type user_id: string - :param user_id: The canonical user id associated with the AWS - account your are granting the permission to. - - :type recursive: boolean - :param recursive: A boolean value to controls whether the command - will apply the grant to all keys within the bucket - or not. The default value is False. By passing a - True value, the call will iterate through all keys - in the bucket and apply the same grant to each key. - CAUTION: If you have a lot of keys, this could take - a long time! - """ - policy = self.get_acl() - policy.acl.add_user_grant(permission, user_id) - self.set_acl(policy, headers=headers) diff --git a/backup/src/boto/s3/multipart.py b/backup/src/boto/s3/multipart.py deleted file mode 100644 index c6160de..0000000 --- a/backup/src/boto/s3/multipart.py +++ /dev/null @@ -1,259 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import user -import key -from boto import handler -import xml.sax - -class CompleteMultiPartUpload(object): - """ - Represents a completed MultiPart Upload. Contains the - following useful attributes: - - * location - The URI of the completed upload - * bucket_name - The name of the bucket in which the upload - is contained - * key_name - The name of the new, completed key - * etag - The MD5 hash of the completed, combined upload - """ - - def __init__(self, bucket=None): - self.bucket = None - self.location = None - self.bucket_name = None - self.key_name = None - self.etag = None - - def __repr__(self): - return '' % (self.bucket_name, - self.key_name) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Location': - self.location = value - elif name == 'Bucket': - self.bucket_name = value - elif name == 'Key': - self.key_name = value - elif name == 'ETag': - self.etag = value - else: - setattr(self, name, value) - -class Part(object): - """ - Represents a single part in a MultiPart upload. - Attributes include: - - * part_number - The integer part number - * last_modified - The last modified date of this part - * etag - The MD5 hash of this part - * size - The size, in bytes, of this part - """ - - def __init__(self, bucket=None): - self.bucket = bucket - self.part_number = None - self.last_modified = None - self.etag = None - self.size = None - - def __repr__(self): - if isinstance(self.part_number, int): - return '' % self.part_number - else: - return '' % None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'PartNumber': - self.part_number = int(value) - elif name == 'LastModified': - self.last_modified = value - elif name == 'ETag': - self.etag = value - elif name == 'Size': - self.size = int(value) - else: - setattr(self, name, value) - -def part_lister(mpupload, part_number_marker=''): - """ - A generator function for listing parts of a multipart upload. - """ - more_results = True - part = None - while more_results: - parts = mpupload.get_all_parts(None, part_number_marker) - for part in parts: - yield part - part_number_marker = mpupload.next_part_number_marker - more_results= mpupload.is_truncated - -class MultiPartUpload(object): - """ - Represents a MultiPart Upload operation. - """ - - def __init__(self, bucket=None): - self.bucket = bucket - self.bucket_name = None - self.key_name = None - self.id = id - self.initiator = None - self.owner = None - self.storage_class = None - self.initiated = None - self.part_number_marker = None - self.next_part_number_marker = None - self.max_parts = None - self.is_truncated = False - self._parts = None - - def __repr__(self): - return '' % self.key_name - - def __iter__(self): - return part_lister(self, part_number_marker=self.part_number_marker) - - def to_xml(self): - self.get_all_parts() - s = '\n' - for part in self: - s += ' \n' - s += ' %d\n' % part.part_number - s += ' %s\n' % part.etag - s += ' \n' - s += '' - return s - - def startElement(self, name, attrs, connection): - if name == 'Initiator': - self.initiator = user.User(self) - return self.initiator - elif name == 'Owner': - self.owner = user.User(self) - return self.owner - elif name == 'Part': - part = Part(self.bucket) - self._parts.append(part) - return part - return None - - def endElement(self, name, value, connection): - if name == 'Bucket': - self.bucket_name = value - elif name == 'Key': - self.key_name = value - elif name == 'UploadId': - self.id = value - elif name == 'StorageClass': - self.storage_class = value - elif name == 'PartNumberMarker': - self.part_number_marker = value - elif name == 'NextPartNumberMarker': - self.next_part_number_marker = value - elif name == 'MaxParts': - self.max_parts = int(value) - elif name == 'IsTruncated': - if value == 'true': - self.is_truncated = True - else: - self.is_truncated = False - else: - setattr(self, name, value) - - def get_all_parts(self, max_parts=None, part_number_marker=None): - """ - Return the uploaded parts of this MultiPart Upload. This is - a lower-level method that requires you to manually page through - results. To simplify this process, you can just use the - object itself as an iterator and it will automatically handle - all of the paging with S3. - """ - self._parts = [] - query_args = 'uploadId=%s' % self.id - if max_parts: - query_args += '&max_parts=%d' % max_parts - if part_number_marker: - query_args += '&part-number-marker=%s' % part_number_marker - response = self.bucket.connection.make_request('GET', self.bucket.name, - self.key_name, - query_args=query_args) - body = response.read() - if response.status == 200: - h = handler.XmlHandler(self, self) - xml.sax.parseString(body, h) - return self._parts - - def upload_part_from_file(self, fp, part_num, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None): - """ - Upload another part of this MultiPart Upload. - - :type fp: file - :param fp: The file object you want to upload. - - :type part_num: int - :param part_num: The number of this part. - - The other parameters are exactly as defined for the - :class:`boto.s3.key.Key` set_contents_from_file method. - """ - if part_num < 1: - raise ValueError('Part numbers must be greater than zero') - query_args = 'uploadId=%s&partNumber=%d' % (self.id, part_num) - key = self.bucket.new_key(self.key_name) - key.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, - md5, reduced_redundancy=False, query_args=query_args) - - def complete_upload(self): - """ - Complete the MultiPart Upload operation. This method should - be called when all parts of the file have been successfully - uploaded to S3. - - :rtype: :class:`boto.s3.multipart.CompletedMultiPartUpload` - :returns: An object representing the completed upload. - """ - xml = self.to_xml() - self.bucket.complete_multipart_upload(self.key_name, - self.id, xml) - - def cancel_upload(self): - """ - Cancels a MultiPart Upload operation. The storage consumed by - any previously uploaded parts will be freed. However, if any - part uploads are currently in progress, those part uploads - might or might not succeed. As a result, it might be necessary - to abort a given multipart upload multiple times in order to - completely free all storage consumed by all parts. - """ - self.bucket.cancel_multipart_upload(self.key_name, self.id) - - diff --git a/backup/src/boto/s3/prefix.py b/backup/src/boto/s3/prefix.py deleted file mode 100644 index fc0f26a..0000000 --- a/backup/src/boto/s3/prefix.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Prefix: - def __init__(self, bucket=None, name=None): - self.bucket = bucket - self.name = name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'Prefix': - self.name = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/s3/resumable_download_handler.py b/backup/src/boto/s3/resumable_download_handler.py deleted file mode 100644 index 0d01477..0000000 --- a/backup/src/boto/s3/resumable_download_handler.py +++ /dev/null @@ -1,330 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import errno -import httplib -import os -import re -import socket -import time -import boto -from boto import config, storage_uri_for_key -from boto.connection import AWSAuthConnection -from boto.exception import ResumableDownloadException -from boto.exception import ResumableTransferDisposition - -""" -Resumable download handler. - -Resumable downloads will retry failed downloads, resuming at the byte count -completed by the last download attempt. If too many retries happen with no -progress (per configurable num_retries param), the download will be aborted. - -The caller can optionally specify a tracker_file_name param in the -ResumableDownloadHandler constructor. If you do this, that file will -save the state needed to allow retrying later, in a separate process -(e.g., in a later run of gsutil). - -Note that resumable downloads work across providers (they depend only -on support Range GETs), but this code is in the boto.s3 package -because it is the wrong abstraction level to go in the top-level boto -package. - -TODO: At some point we should refactor the code to have a storage_service -package where all these provider-independent files go. -""" - - -class ByteTranslatingCallbackHandler(object): - """ - Proxy class that translates progress callbacks made by - boto.s3.Key.get_file(), taking into account that we're resuming - a download. - """ - def __init__(self, proxied_cb, download_start_point): - self.proxied_cb = proxied_cb - self.download_start_point = download_start_point - - def call(self, total_bytes_uploaded, total_size): - self.proxied_cb(self.download_start_point + total_bytes_uploaded, - self.download_start_point + total_size) - - -def get_cur_file_size(fp, position_to_eof=False): - """ - Returns size of file, optionally leaving fp positioned at EOF. - """ - if not position_to_eof: - cur_pos = fp.tell() - fp.seek(0, os.SEEK_END) - cur_file_size = fp.tell() - if not position_to_eof: - fp.seek(cur_pos, os.SEEK_SET) - return cur_file_size - - -class ResumableDownloadHandler(object): - """ - Handler for resumable downloads. - """ - - ETAG_REGEX = '([a-z0-9]{32})\n' - - RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error, - socket.gaierror) - - def __init__(self, tracker_file_name=None, num_retries=None): - """ - Constructor. Instantiate once for each downloaded file. - - :type tracker_file_name: string - :param tracker_file_name: optional file name to save tracking info - about this download. If supplied and the current process fails - the download, it can be retried in a new process. If called - with an existing file containing an unexpired timestamp, - we'll resume the transfer for this file; else we'll start a - new resumable download. - - :type num_retries: int - :param num_retries: the number of times we'll re-try a resumable - download making no progress. (Count resets every time we get - progress, so download can span many more than this number of - retries.) - """ - self.tracker_file_name = tracker_file_name - self.num_retries = num_retries - self.etag_value_for_current_download = None - if tracker_file_name: - self._load_tracker_file_etag() - # Save download_start_point in instance state so caller can - # find how much was transferred by this ResumableDownloadHandler - # (across retries). - self.download_start_point = None - - def _load_tracker_file_etag(self): - f = None - try: - f = open(self.tracker_file_name, 'r') - etag_line = f.readline() - m = re.search(self.ETAG_REGEX, etag_line) - if m: - self.etag_value_for_current_download = m.group(1) - else: - print('Couldn\'t read etag in tracker file (%s). Restarting ' - 'download from scratch.' % self.tracker_file_name) - except IOError, e: - # Ignore non-existent file (happens first time a download - # is attempted on an object), but warn user for other errors. - if e.errno != errno.ENOENT: - # Will restart because - # self.etag_value_for_current_download == None. - print('Couldn\'t read URI tracker file (%s): %s. Restarting ' - 'download from scratch.' % - (self.tracker_file_name, e.strerror)) - finally: - if f: - f.close() - - def _save_tracker_info(self, key): - self.etag_value_for_current_download = key.etag.strip('"\'') - if not self.tracker_file_name: - return - f = None - try: - f = open(self.tracker_file_name, 'w') - f.write('%s\n' % self.etag_value_for_current_download) - except IOError, e: - raise ResumableDownloadException( - 'Couldn\'t write tracker file (%s): %s.\nThis can happen' - 'if you\'re using an incorrectly configured download tool\n' - '(e.g., gsutil configured to save tracker files to an ' - 'unwritable directory)' % - (self.tracker_file_name, e.strerror), - ResumableTransferDisposition.ABORT) - finally: - if f: - f.close() - - def _remove_tracker_file(self): - if (self.tracker_file_name and - os.path.exists(self.tracker_file_name)): - os.unlink(self.tracker_file_name) - - def _attempt_resumable_download(self, key, fp, headers, cb, num_cb, - torrent, version_id): - """ - Attempts a resumable download. - - Raises ResumableDownloadException if any problems occur. - """ - cur_file_size = get_cur_file_size(fp, position_to_eof=True) - - if (cur_file_size and - self.etag_value_for_current_download and - self.etag_value_for_current_download == key.etag.strip('"\'')): - # Try to resume existing transfer. - if cur_file_size > key.size: - raise ResumableDownloadException( - '%s is larger (%d) than %s (%d).\nDeleting tracker file, so ' - 'if you re-try this download it will start from scratch' % - (fp.name, cur_file_size, str(storage_uri_for_key(key)), - key.size), ResumableTransferDisposition.ABORT) - elif cur_file_size == key.size: - if key.bucket.connection.debug >= 1: - print 'Download complete.' - return - if key.bucket.connection.debug >= 1: - print 'Resuming download.' - headers = headers.copy() - headers['Range'] = 'bytes=%d-%d' % (cur_file_size, key.size - 1) - cb = ByteTranslatingCallbackHandler(cb, cur_file_size).call - self.download_start_point = cur_file_size - else: - if key.bucket.connection.debug >= 1: - print 'Starting new resumable download.' - self._save_tracker_info(key) - self.download_start_point = 0 - # Truncate the file, in case a new resumable download is being - # started atop an existing file. - fp.truncate(0) - - # Disable AWSAuthConnection-level retry behavior, since that would - # cause downloads to restart from scratch. - key.get_file(fp, headers, cb, num_cb, torrent, version_id, - override_num_retries=0) - fp.flush() - - def _check_final_md5(self, key, file_name): - """ - Checks that etag from server agrees with md5 computed after the - download completes. This is important, since the download could - have spanned a number of hours and multiple processes (e.g., - gsutil runs), and the user could change some of the file and not - realize they have inconsistent data. - """ - fp = open(file_name, 'r') - if key.bucket.connection.debug >= 1: - print 'Checking md5 against etag.' - hex_md5 = key.compute_md5(fp)[0] - if hex_md5 != key.etag.strip('"\''): - file_name = fp.name - fp.close() - os.unlink(file_name) - raise ResumableDownloadException( - 'File changed during download: md5 signature doesn\'t match ' - 'etag (incorrect downloaded file deleted)', - ResumableTransferDisposition.ABORT) - - def get_file(self, key, fp, headers, cb=None, num_cb=10, torrent=False, - version_id=None): - """ - Retrieves a file from a Key - :type key: :class:`boto.s3.key.Key` or subclass - :param key: The Key object from which upload is to be downloaded - - :type fp: file - :param fp: File pointer into which data should be downloaded - - :type headers: string - :param: headers to send when retrieving the files - - :type cb: function - :param cb: (optional) a callback function that will be called to report - progress on the download. The callback should accept two integer - parameters, the first representing the number of bytes that have - been successfully transmitted from the storage service and - the second representing the total number of bytes that need - to be transmitted. - - :type num_cb: int - :param num_cb: (optional) If a callback is specified with the cb - parameter this parameter determines the granularity of the callback - by defining the maximum number of times the callback will be - called during the file transfer. - - :type torrent: bool - :param torrent: Flag for whether to get a torrent for the file - - :type version_id: string - :param version_id: The version ID (optional) - - Raises ResumableDownloadException if a problem occurs during - the transfer. - """ - - debug = key.bucket.connection.debug - if not headers: - headers = {} - - # Use num-retries from constructor if one was provided; else check - # for a value specified in the boto config file; else default to 5. - if self.num_retries is None: - self.num_retries = config.getint('Boto', 'num_retries', 5) - progress_less_iterations = 0 - - while True: # Retry as long as we're making progress. - had_file_bytes_before_attempt = get_cur_file_size(fp) - try: - self._attempt_resumable_download(key, fp, headers, cb, num_cb, - torrent, version_id) - # Download succceded, so remove the tracker file (if have one). - self._remove_tracker_file() - self._check_final_md5(key, fp.name) - if debug >= 1: - print 'Resumable download complete.' - return - except self.RETRYABLE_EXCEPTIONS, e: - if debug >= 1: - print('Caught exception (%s)' % e.__repr__()) - except ResumableDownloadException, e: - if e.disposition == ResumableTransferDisposition.ABORT: - if debug >= 1: - print('Caught non-retryable ResumableDownloadException ' - '(%s)' % e.message) - raise - else: - if debug >= 1: - print('Caught ResumableDownloadException (%s) - will ' - 'retry' % e.message) - - # At this point we had a re-tryable failure; see if made progress. - if get_cur_file_size(fp) > had_file_bytes_before_attempt: - progress_less_iterations = 0 - else: - progress_less_iterations += 1 - - if progress_less_iterations > self.num_retries: - # Don't retry any longer in the current process. - raise ResumableDownloadException( - 'Too many resumable download attempts failed without ' - 'progress. You might try this download again later', - ResumableTransferDisposition.ABORT) - - # Close the key, in case a previous download died partway - # through and left data in the underlying key HTTP buffer. - key.close() - - sleep_time_secs = 2**progress_less_iterations - if debug >= 1: - print('Got retryable failure (%d progress-less in a row).\n' - 'Sleeping %d seconds before re-trying' % - (progress_less_iterations, sleep_time_secs)) - time.sleep(sleep_time_secs) diff --git a/backup/src/boto/s3/user.py b/backup/src/boto/s3/user.py deleted file mode 100644 index f45f038..0000000 --- a/backup/src/boto/s3/user.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class User: - def __init__(self, parent=None, id='', display_name=''): - if parent: - parent.owner = self - self.type = None - self.id = id - self.display_name = display_name - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'DisplayName': - self.display_name = value - elif name == 'ID': - self.id = value - else: - setattr(self, name, value) - - def to_xml(self, element_name='Owner'): - if self.type: - s = '<%s xsi:type="%s">' % (element_name, self.type) - else: - s = '<%s>' % element_name - s += '%s' % self.id - s += '%s' % self.display_name - s += '' % element_name - return s diff --git a/backup/src/boto/sdb/__init__.py b/backup/src/boto/sdb/__init__.py deleted file mode 100644 index f5642c1..0000000 --- a/backup/src/boto/sdb/__init__.py +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from regioninfo import SDBRegionInfo - -def regions(): - """ - Get all available regions for the SDB service. - - :rtype: list - :return: A list of :class:`boto.sdb.regioninfo.RegionInfo` instances - """ - return [SDBRegionInfo(name='us-east-1', - endpoint='sdb.amazonaws.com'), - SDBRegionInfo(name='eu-west-1', - endpoint='sdb.eu-west-1.amazonaws.com'), - SDBRegionInfo(name='us-west-1', - endpoint='sdb.us-west-1.amazonaws.com'), - SDBRegionInfo(name='ap-southeast-1', - endpoint='sdb.ap-southeast-1.amazonaws.com') - ] - -def connect_to_region(region_name): - """ - Given a valid region name, return a - :class:`boto.sdb.connection.SDBConnection`. - - :param str region_name: The name of the region to connect to. - - :rtype: :class:`boto.sdb.connection.SDBConnection` or ``None`` - :return: A connection to the given region, or None if an invalid region - name is given - """ - for region in regions(): - if region.name == region_name: - return region.connect() - return None diff --git a/backup/src/boto/sdb/connection.py b/backup/src/boto/sdb/connection.py deleted file mode 100644 index b5a45b8..0000000 --- a/backup/src/boto/sdb/connection.py +++ /dev/null @@ -1,607 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import xml.sax -import threading -from boto import handler -from boto.connection import AWSQueryConnection -from boto.sdb.domain import Domain, DomainMetaData -from boto.sdb.item import Item -from boto.sdb.regioninfo import SDBRegionInfo -from boto.exception import SDBResponseError - -class ItemThread(threading.Thread): - """ - A threaded :class:`Item ` retriever utility class. - Retrieved :class:`Item ` objects are stored in the - ``items`` instance variable after - :py:meth:`run() ` is called. - - .. tip:: - The item retrieval will not start until the - :func:`run() ` method is called. - """ - def __init__(self, name, domain_name, item_names): - """ - :param str name: A thread name. Used for identification. - :param str domain_name: The name of a SimpleDB - :class:`Domain ` - :type item_names: string or list of strings - :param item_names: The name(s) of the items to retrieve from the specified - :class:`Domain `. - :ivar list items: A list of items retrieved. Starts as empty list. - """ - threading.Thread.__init__(self, name=name) - #print 'starting %s with %d items' % (name, len(item_names)) - self.domain_name = domain_name - self.conn = SDBConnection() - self.item_names = item_names - self.items = [] - - def run(self): - """ - Start the threaded retrieval of items. Populates the - ``items`` list with :class:`Item ` objects. - """ - for item_name in self.item_names: - item = self.conn.get_attributes(self.domain_name, item_name) - self.items.append(item) - -#boto.set_stream_logger('sdb') - -class SDBConnection(AWSQueryConnection): - """ - This class serves as a gateway to your SimpleDB region (defaults to - us-east-1). Methods within allow access to SimpleDB - :class:`Domain ` objects and their associated - :class:`Item ` objects. - - .. tip:: - While you may instantiate this class directly, it may be easier to - go through :py:func:`boto.connect_sdb`. - """ - DefaultRegionName = 'us-east-1' - DefaultRegionEndpoint = 'sdb.amazonaws.com' - APIVersion = '2009-04-15' - ResponseError = SDBResponseError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/', - converter=None): - """ - For any keywords that aren't documented, refer to the parent class, - :py:class:`boto.connection.AWSAuthConnection`. You can avoid having - to worry about these keyword arguments by instantiating these objects - via :py:func:`boto.connect_sdb`. - - :type region: :class:`boto.sdb.regioninfo.SDBRegionInfo` - :keyword region: Explicitly specify a region. Defaults to ``us-east-1`` - if not specified. - """ - if not region: - region = SDBRegionInfo(self, self.DefaultRegionName, - self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, - aws_secret_access_key, - is_secure, port, proxy, - proxy_port, proxy_user, proxy_pass, - self.region.endpoint, debug, - https_connection_factory, path) - self.box_usage = 0.0 - self.converter = converter - self.item_cls = Item - - def _required_auth_capability(self): - return ['sdb'] - - def set_item_cls(self, cls): - """ - While the default item class is :py:class:`boto.sdb.item.Item`, this - default may be overridden. Use this method to change a connection's - item class. - - :param object cls: The new class to set as this connection's item - class. See the default item class for inspiration as to what your - replacement should/could look like. - """ - self.item_cls = cls - - def _build_name_value_list(self, params, attributes, replace=False, - label='Attribute'): - keys = attributes.keys() - keys.sort() - i = 1 - for key in keys: - value = attributes[key] - if isinstance(value, list): - for v in value: - params['%s.%d.Name' % (label, i)] = key - if self.converter: - v = self.converter.encode(v) - params['%s.%d.Value' % (label, i)] = v - if replace: - params['%s.%d.Replace' % (label, i)] = 'true' - i += 1 - else: - params['%s.%d.Name' % (label, i)] = key - if self.converter: - value = self.converter.encode(value) - params['%s.%d.Value' % (label, i)] = value - if replace: - params['%s.%d.Replace' % (label, i)] = 'true' - i += 1 - - def _build_expected_value(self, params, expected_value): - params['Expected.1.Name'] = expected_value[0] - if expected_value[1] is True: - params['Expected.1.Exists'] = 'true' - elif expected_value[1] is False: - params['Expected.1.Exists'] = 'false' - else: - params['Expected.1.Value'] = expected_value[1] - - def _build_batch_list(self, params, items, replace=False): - item_names = items.keys() - i = 0 - for item_name in item_names: - params['Item.%d.ItemName' % i] = item_name - j = 0 - item = items[item_name] - if item is not None: - attr_names = item.keys() - for attr_name in attr_names: - value = item[attr_name] - if isinstance(value, list): - for v in value: - if self.converter: - v = self.converter.encode(v) - params['Item.%d.Attribute.%d.Name' % (i, j)] = attr_name - params['Item.%d.Attribute.%d.Value' % (i, j)] = v - if replace: - params['Item.%d.Attribute.%d.Replace' % (i, j)] = 'true' - j += 1 - else: - params['Item.%d.Attribute.%d.Name' % (i, j)] = attr_name - if self.converter: - value = self.converter.encode(value) - params['Item.%d.Attribute.%d.Value' % (i, j)] = value - if replace: - params['Item.%d.Attribute.%d.Replace' % (i, j)] = 'true' - j += 1 - i += 1 - - def _build_name_list(self, params, attribute_names): - i = 1 - attribute_names.sort() - for name in attribute_names: - params['Attribute.%d.Name' % i] = name - i += 1 - - def get_usage(self): - """ - Returns the BoxUsage (in USD) accumulated on this specific SDBConnection - instance. - - .. tip:: This can be out of date, and should only be treated as a - rough estimate. Also note that this estimate only applies to the - requests made on this specific connection instance. It is by - no means an account-wide estimate. - - :rtype: float - :return: The accumulated BoxUsage of all requests made on the connection. - """ - return self.box_usage - - def print_usage(self): - """ - Print the BoxUsage and approximate costs of all requests made on - this specific SDBConnection instance. - - .. tip:: This can be out of date, and should only be treated as a - rough estimate. Also note that this estimate only applies to the - requests made on this specific connection instance. It is by - no means an account-wide estimate. - """ - print 'Total Usage: %f compute seconds' % self.box_usage - cost = self.box_usage * 0.14 - print 'Approximate Cost: $%f' % cost - - def get_domain(self, domain_name, validate=True): - """ - Retrieves a :py:class:`boto.sdb.domain.Domain` object whose name - matches ``domain_name``. - - :param str domain_name: The name of the domain to retrieve - :keyword bool validate: When ``True``, check to see if the domain - actually exists. If ``False``, blindly return a - :py:class:`Domain ` object with the - specified name set. - - :raises: - :py:class:`boto.exception.SDBResponseError` if ``validate`` is - ``True`` and no match could be found. - - :rtype: :py:class:`boto.sdb.domain.Domain` - :return: The requested domain - """ - domain = Domain(self, domain_name) - if validate: - self.select(domain, """select * from `%s` limit 1""" % domain_name) - return domain - - def lookup(self, domain_name, validate=True): - """ - Lookup an existing SimpleDB domain. This differs from - :py:meth:`get_domain` in that ``None`` is returned if ``validate`` is - ``True`` and no match was found (instead of raising an exception). - - :param str domain_name: The name of the domain to retrieve - - :param bool validate: If ``True``, a ``None`` value will be returned - if the specified domain can't be found. If ``False``, a - :py:class:`Domain ` object will be dumbly - returned, regardless of whether it actually exists. - - :rtype: :class:`boto.sdb.domain.Domain` object or ``None`` - :return: The Domain object or ``None`` if the domain does not exist. - """ - try: - domain = self.get_domain(domain_name, validate) - except: - domain = None - return domain - - def get_all_domains(self, max_domains=None, next_token=None): - """ - Returns a :py:class:`boto.resultset.ResultSet` containing - all :py:class:`boto.sdb.domain.Domain` objects associated with - this connection's Access Key ID. - - :keyword int max_domains: Limit the returned - :py:class:`ResultSet ` to the specified - number of members. - :keyword str next_token: A token string that was returned in an - earlier call to this method as the ``next_token`` attribute - on the returned :py:class:`ResultSet ` - object. This attribute is set if there are more than Domains than - the value specified in the ``max_domains`` keyword. Pass the - ``next_token`` value from you earlier query in this keyword to - get the next 'page' of domains. - """ - params = {} - if max_domains: - params['MaxNumberOfDomains'] = max_domains - if next_token: - params['NextToken'] = next_token - return self.get_list('ListDomains', params, [('DomainName', Domain)]) - - def create_domain(self, domain_name): - """ - Create a SimpleDB domain. - - :type domain_name: string - :param domain_name: The name of the new domain - - :rtype: :class:`boto.sdb.domain.Domain` object - :return: The newly created domain - """ - params = {'DomainName':domain_name} - d = self.get_object('CreateDomain', params, Domain) - d.name = domain_name - return d - - def get_domain_and_name(self, domain_or_name): - """ - Given a ``str`` or :class:`boto.sdb.domain.Domain`, return a - ``tuple`` with the following members (in order): - - * In instance of :class:`boto.sdb.domain.Domain` for the requested - domain - * The domain's name as a ``str`` - - :type domain_or_name: ``str`` or :class:`boto.sdb.domain.Domain` - :param domain_or_name: The domain or domain name to get the domain - and name for. - - :raises: :class:`boto.exception.SDBResponseError` when an invalid - domain name is specified. - - :rtype: tuple - :return: A ``tuple`` with contents outlined as per above. - """ - if (isinstance(domain_or_name, Domain)): - return (domain_or_name, domain_or_name.name) - else: - return (self.get_domain(domain_or_name), domain_or_name) - - def delete_domain(self, domain_or_name): - """ - Delete a SimpleDB domain. - - .. caution:: This will delete the domain and all items within the domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :rtype: bool - :return: True if successful - - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName':domain_name} - return self.get_status('DeleteDomain', params) - - def domain_metadata(self, domain_or_name): - """ - Get the Metadata for a SimpleDB domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :rtype: :class:`boto.sdb.domain.DomainMetaData` object - :return: The newly created domain metadata object - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName':domain_name} - d = self.get_object('DomainMetadata', params, DomainMetaData) - d.domain = domain - return d - - def put_attributes(self, domain_or_name, item_name, attributes, - replace=True, expected_value=None): - """ - Store attributes for a given item in a domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :type item_name: string - :param item_name: The name of the item whose attributes are being - stored. - - :type attribute_names: dict or dict-like object - :param attribute_names: The name/value pairs to store as attributes - - :type expected_value: list - :param expected_value: If supplied, this is a list or tuple consisting - of a single attribute name and expected value. The list can be - of the form: - - * ['name', 'value'] - - In which case the call will first verify that the attribute "name" - of this item has a value of "value". If it does, the delete - will proceed, otherwise a ConditionalCheckFailed error will be - returned. The list can also be of the form: - - * ['name', True|False] - - which will simply check for the existence (True) or - non-existence (False) of the attribute. - - :type replace: bool - :param replace: Whether the attribute values passed in will replace - existing values or will be added as addition values. - Defaults to True. - - :rtype: bool - :return: True if successful - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName' : domain_name, - 'ItemName' : item_name} - self._build_name_value_list(params, attributes, replace) - if expected_value: - self._build_expected_value(params, expected_value) - return self.get_status('PutAttributes', params) - - def batch_put_attributes(self, domain_or_name, items, replace=True): - """ - Store attributes for multiple items in a domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :type items: dict or dict-like object - :param items: A dictionary-like object. The keys of the dictionary are - the item names and the values are themselves dictionaries - of attribute names/values, exactly the same as the - attribute_names parameter of the scalar put_attributes - call. - - :type replace: bool - :param replace: Whether the attribute values passed in will replace - existing values or will be added as addition values. - Defaults to True. - - :rtype: bool - :return: True if successful - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName' : domain_name} - self._build_batch_list(params, items, replace) - return self.get_status('BatchPutAttributes', params, verb='POST') - - def get_attributes(self, domain_or_name, item_name, attribute_names=None, - consistent_read=False, item=None): - """ - Retrieve attributes for a given item in a domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :type item_name: string - :param item_name: The name of the item whose attributes are - being retrieved. - - :type attribute_names: string or list of strings - :param attribute_names: An attribute name or list of attribute names. - This parameter is optional. If not supplied, all attributes will - be retrieved for the item. - - :type consistent_read: bool - :param consistent_read: When set to true, ensures that the most recent - data is returned. - - :type item: :class:`boto.sdb.item.Item` - :keyword item: Instead of instantiating a new Item object, you may - specify one to update. - - :rtype: :class:`boto.sdb.item.Item` - :return: An Item with the requested attribute name/values set on it - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName' : domain_name, - 'ItemName' : item_name} - if consistent_read: - params['ConsistentRead'] = 'true' - if attribute_names: - if not isinstance(attribute_names, list): - attribute_names = [attribute_names] - self.build_list_params(params, attribute_names, 'AttributeName') - response = self.make_request('GetAttributes', params) - body = response.read() - if response.status == 200: - if item == None: - item = self.item_cls(domain, item_name) - h = handler.XmlHandler(item, self) - xml.sax.parseString(body, h) - return item - else: - raise SDBResponseError(response.status, response.reason, body) - - def delete_attributes(self, domain_or_name, item_name, attr_names=None, - expected_value=None): - """ - Delete attributes from a given item in a domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :type item_name: string - :param item_name: The name of the item whose attributes are being - deleted. - - :type attributes: dict, list or :class:`boto.sdb.item.Item` - :param attributes: Either a list containing attribute names which - will cause all values associated with that attribute - name to be deleted or a dict or Item containing the - attribute names and keys and list of values to - delete as the value. If no value is supplied, - all attribute name/values for the item will be - deleted. - - :type expected_value: list - :param expected_value: If supplied, this is a list or tuple consisting - of a single attribute name and expected value. The list can be - of the form: - - * ['name', 'value'] - - In which case the call will first verify that the attribute "name" - of this item has a value of "value". If it does, the delete - will proceed, otherwise a ConditionalCheckFailed error will be - returned. The list can also be of the form: - - * ['name', True|False] - - which will simply check for the existence (True) or - non-existence (False) of the attribute. - - :rtype: bool - :return: True if successful - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName':domain_name, - 'ItemName' : item_name} - if attr_names: - if isinstance(attr_names, list): - self._build_name_list(params, attr_names) - elif isinstance(attr_names, dict) or isinstance(attr_names, self.item_cls): - self._build_name_value_list(params, attr_names) - if expected_value: - self._build_expected_value(params, expected_value) - return self.get_status('DeleteAttributes', params) - - def batch_delete_attributes(self, domain_or_name, items): - """ - Delete multiple items in a domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. - :param domain_or_name: Either the name of a domain or a Domain object - - :type items: dict or dict-like object - :param items: A dictionary-like object. The keys of the dictionary are - the item names and the values are either: - - * dictionaries of attribute names/values, exactly the - same as the attribute_names parameter of the scalar - put_attributes call. The attribute name/value pairs - will only be deleted if they match the name/value - pairs passed in. - * None which means that all attributes associated - with the item should be deleted. - - :return: True if successful - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'DomainName' : domain_name} - self._build_batch_list(params, items, False) - return self.get_status('BatchDeleteAttributes', params, verb='POST') - - def select(self, domain_or_name, query='', next_token=None, - consistent_read=False): - """ - Returns a set of Attributes for item names within domain_name that - match the query. The query must be expressed in using the SELECT - style syntax rather than the original SimpleDB query language. - Even though the select request does not require a domain object, - a domain object must be passed into this method so the Item objects - returned can point to the appropriate domain. - - :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object - :param domain_or_name: Either the name of a domain or a Domain object - - :type query: string - :param query: The SimpleDB query to be performed. - - :type consistent_read: bool - :param consistent_read: When set to true, ensures that the most recent - data is returned. - - :rtype: ResultSet - :return: An iterator containing the results. - """ - domain, domain_name = self.get_domain_and_name(domain_or_name) - params = {'SelectExpression' : query} - if consistent_read: - params['ConsistentRead'] = 'true' - if next_token: - params['NextToken'] = next_token - try: - return self.get_list('Select', params, [('Item', self.item_cls)], - parent=domain) - except SDBResponseError, e: - e.body = "Query: %s\n%s" % (query, e.body) - raise e diff --git a/backup/src/boto/sdb/db/__init__.py b/backup/src/boto/sdb/db/__init__.py deleted file mode 100644 index 86044ed..0000000 --- a/backup/src/boto/sdb/db/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - diff --git a/backup/src/boto/sdb/db/blob.py b/backup/src/boto/sdb/db/blob.py deleted file mode 100644 index 45a3624..0000000 --- a/backup/src/boto/sdb/db/blob.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - - -class Blob(object): - """Blob object""" - def __init__(self, value=None, file=None, id=None): - self._file = file - self.id = id - self.value = value - - @property - def file(self): - from StringIO import StringIO - if self._file: - f = self._file - else: - f = StringIO(self.value) - return f - - def __str__(self): - if hasattr(self.file, "get_contents_as_string"): - value = self.file.get_contents_as_string() - else: - value = self.file.getvalue() - try: - return str(value) - except: - return unicode(value) - - def read(self): - return self.file.read() - - def readline(self): - return self.file.readline() - - def next(self): - return self.file.next() - - def __iter__(self): - return iter(self.file) - - @property - def size(self): - if self._file: - return self._file.size - elif self.value: - return len(self.value) - else: - return 0 diff --git a/backup/src/boto/sdb/db/key.py b/backup/src/boto/sdb/db/key.py deleted file mode 100644 index 42a9d8d..0000000 --- a/backup/src/boto/sdb/db/key.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Key(object): - - @classmethod - def from_path(cls, *args, **kwds): - raise NotImplementedError, "Paths are not currently supported" - - def __init__(self, encoded=None, obj=None): - self.name = None - if obj: - self.id = obj.id - self.kind = obj.kind() - else: - self.id = None - self.kind = None - - def app(self): - raise NotImplementedError, "Applications are not currently supported" - - def kind(self): - return self.kind - - def id(self): - return self.id - - def name(self): - raise NotImplementedError, "Key Names are not currently supported" - - def id_or_name(self): - return self.id - - def has_id_or_name(self): - return self.id != None - - def parent(self): - raise NotImplementedError, "Key parents are not currently supported" - - def __str__(self): - return self.id_or_name() diff --git a/backup/src/boto/sdb/db/manager/__init__.py b/backup/src/boto/sdb/db/manager/__init__.py deleted file mode 100644 index 0777796..0000000 --- a/backup/src/boto/sdb/db/manager/__init__.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -import boto - -def get_manager(cls): - """ - Returns the appropriate Manager class for a given Model class. It does this by - looking in the boto config for a section like this:: - - [DB] - db_type = SimpleDB - db_user = - db_passwd = - db_name = my_domain - [DB_TestBasic] - db_type = SimpleDB - db_user = - db_passwd = - db_name = basic_domain - db_port = 1111 - - The values in the DB section are "generic values" that will be used if nothing more - specific is found. You can also create a section for a specific Model class that - gives the db info for that class. In the example above, TestBasic is a Model subclass. - """ - db_user = boto.config.get('DB', 'db_user', None) - db_passwd = boto.config.get('DB', 'db_passwd', None) - db_type = boto.config.get('DB', 'db_type', 'SimpleDB') - db_name = boto.config.get('DB', 'db_name', None) - db_table = boto.config.get('DB', 'db_table', None) - db_host = boto.config.get('DB', 'db_host', "sdb.amazonaws.com") - db_port = boto.config.getint('DB', 'db_port', 443) - enable_ssl = boto.config.getbool('DB', 'enable_ssl', True) - sql_dir = boto.config.get('DB', 'sql_dir', None) - debug = boto.config.getint('DB', 'debug', 0) - # first see if there is a fully qualified section name in the Boto config file - module_name = cls.__module__.replace('.', '_') - db_section = 'DB_' + module_name + '_' + cls.__name__ - if not boto.config.has_section(db_section): - db_section = 'DB_' + cls.__name__ - if boto.config.has_section(db_section): - db_user = boto.config.get(db_section, 'db_user', db_user) - db_passwd = boto.config.get(db_section, 'db_passwd', db_passwd) - db_type = boto.config.get(db_section, 'db_type', db_type) - db_name = boto.config.get(db_section, 'db_name', db_name) - db_table = boto.config.get(db_section, 'db_table', db_table) - db_host = boto.config.get(db_section, 'db_host', db_host) - db_port = boto.config.getint(db_section, 'db_port', db_port) - enable_ssl = boto.config.getint(db_section, 'enable_ssl', enable_ssl) - debug = boto.config.getint(db_section, 'debug', debug) - elif hasattr(cls.__bases__[0], "_manager"): - return cls.__bases__[0]._manager - if db_type == 'SimpleDB': - from sdbmanager import SDBManager - return SDBManager(cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, sql_dir, enable_ssl) - elif db_type == 'PostgreSQL': - from pgmanager import PGManager - if db_table: - return PGManager(cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, sql_dir, enable_ssl) - else: - return None - elif db_type == 'XML': - from xmlmanager import XMLManager - return XMLManager(cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, sql_dir, enable_ssl) - else: - raise ValueError, 'Unknown db_type: %s' % db_type - diff --git a/backup/src/boto/sdb/db/manager/pgmanager.py b/backup/src/boto/sdb/db/manager/pgmanager.py deleted file mode 100644 index 73a93f0..0000000 --- a/backup/src/boto/sdb/db/manager/pgmanager.py +++ /dev/null @@ -1,389 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -from boto.sdb.db.key import Key -from boto.sdb.db.model import Model -import psycopg2 -import psycopg2.extensions -import uuid -import os -import string -from boto.exception import SDBPersistenceError - -psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) - -class PGConverter: - - def __init__(self, manager): - self.manager = manager - self.type_map = {Key : (self.encode_reference, self.decode_reference), - Model : (self.encode_reference, self.decode_reference)} - - def encode(self, type, value): - if type in self.type_map: - encode = self.type_map[type][0] - return encode(value) - return value - - def decode(self, type, value): - if type in self.type_map: - decode = self.type_map[type][1] - return decode(value) - return value - - def encode_prop(self, prop, value): - if isinstance(value, list): - if hasattr(prop, 'item_type'): - s = "{" - new_value = [] - for v in value: - item_type = getattr(prop, 'item_type') - if Model in item_type.mro(): - item_type = Model - new_value.append('%s' % self.encode(item_type, v)) - s += ','.join(new_value) - s += "}" - return s - else: - return value - return self.encode(prop.data_type, value) - - def decode_prop(self, prop, value): - if prop.data_type == list: - if value != None: - if not isinstance(value, list): - value = [value] - if hasattr(prop, 'item_type'): - item_type = getattr(prop, "item_type") - if Model in item_type.mro(): - if item_type != self.manager.cls: - return item_type._manager.decode_value(prop, value) - else: - item_type = Model - return [self.decode(item_type, v) for v in value] - return value - elif hasattr(prop, 'reference_class'): - ref_class = getattr(prop, 'reference_class') - if ref_class != self.manager.cls: - return ref_class._manager.decode_value(prop, value) - else: - return self.decode(prop.data_type, value) - elif hasattr(prop, 'calculated_type'): - calc_type = getattr(prop, 'calculated_type') - return self.decode(calc_type, value) - else: - return self.decode(prop.data_type, value) - - def encode_reference(self, value): - if isinstance(value, str) or isinstance(value, unicode): - return value - if value == None: - return '' - else: - return value.id - - def decode_reference(self, value): - if not value: - return None - try: - return self.manager.get_object_from_id(value) - except: - raise ValueError, 'Unable to convert %s to Object' % value - -class PGManager(object): - - def __init__(self, cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, sql_dir, enable_ssl): - self.cls = cls - self.db_name = db_name - self.db_user = db_user - self.db_passwd = db_passwd - self.db_host = db_host - self.db_port = db_port - self.db_table = db_table - self.sql_dir = sql_dir - self.in_transaction = False - self.converter = PGConverter(self) - self._connect() - - def _build_connect_string(self): - cs = 'dbname=%s user=%s password=%s host=%s port=%d' - return cs % (self.db_name, self.db_user, self.db_passwd, - self.db_host, self.db_port) - - def _connect(self): - self.connection = psycopg2.connect(self._build_connect_string()) - self.connection.set_client_encoding('UTF8') - self.cursor = self.connection.cursor() - - def _object_lister(self, cursor): - try: - for row in cursor: - yield self._object_from_row(row, cursor.description) - except StopIteration: - cursor.close() - raise StopIteration - - def _dict_from_row(self, row, description): - d = {} - for i in range(0, len(row)): - d[description[i][0]] = row[i] - return d - - def _object_from_row(self, row, description=None): - if not description: - description = self.cursor.description - d = self._dict_from_row(row, description) - obj = self.cls(d['id']) - obj._manager = self - obj._auto_update = False - for prop in obj.properties(hidden=False): - if prop.data_type != Key: - v = self.decode_value(prop, d[prop.name]) - v = prop.make_value_from_datastore(v) - if hasattr(prop, 'calculated_type'): - prop._set_direct(obj, v) - elif not prop.empty(v): - setattr(obj, prop.name, v) - else: - setattr(obj, prop.name, prop.default_value()) - return obj - - def _build_insert_qs(self, obj, calculated): - fields = [] - values = [] - templs = [] - id_calculated = [p for p in calculated if p.name == 'id'] - for prop in obj.properties(hidden=False): - if prop not in calculated: - value = prop.get_value_for_datastore(obj) - if value != prop.default_value() or prop.required: - value = self.encode_value(prop, value) - values.append(value) - fields.append('"%s"' % prop.name) - templs.append('%s') - qs = 'INSERT INTO "%s" (' % self.db_table - if len(id_calculated) == 0: - qs += '"id",' - qs += ','.join(fields) - qs += ") VALUES (" - if len(id_calculated) == 0: - qs += "'%s'," % obj.id - qs += ','.join(templs) - qs += ')' - if calculated: - qs += ' RETURNING ' - calc_values = ['"%s"' % p.name for p in calculated] - qs += ','.join(calc_values) - qs += ';' - return qs, values - - def _build_update_qs(self, obj, calculated): - fields = [] - values = [] - for prop in obj.properties(hidden=False): - if prop not in calculated: - value = prop.get_value_for_datastore(obj) - if value != prop.default_value() or prop.required: - value = self.encode_value(prop, value) - values.append(value) - field = '"%s"=' % prop.name - field += '%s' - fields.append(field) - qs = 'UPDATE "%s" SET ' % self.db_table - qs += ','.join(fields) - qs += """ WHERE "id" = '%s'""" % obj.id - if calculated: - qs += ' RETURNING ' - calc_values = ['"%s"' % p.name for p in calculated] - qs += ','.join(calc_values) - qs += ';' - return qs, values - - def _get_sql(self, mapping=None): - print '_get_sql' - sql = None - if self.sql_dir: - path = os.path.join(self.sql_dir, self.cls.__name__ + '.sql') - print path - if os.path.isfile(path): - fp = open(path) - sql = fp.read() - fp.close() - t = string.Template(sql) - sql = t.safe_substitute(mapping) - return sql - - def start_transaction(self): - print 'start_transaction' - self.in_transaction = True - - def end_transaction(self): - print 'end_transaction' - self.in_transaction = False - self.commit() - - def commit(self): - if not self.in_transaction: - print '!!commit on %s' % self.db_table - try: - self.connection.commit() - - except psycopg2.ProgrammingError, err: - self.connection.rollback() - raise err - - def rollback(self): - print '!!rollback on %s' % self.db_table - self.connection.rollback() - - def delete_table(self): - self.cursor.execute('DROP TABLE "%s";' % self.db_table) - self.commit() - - def create_table(self, mapping=None): - self.cursor.execute(self._get_sql(mapping)) - self.commit() - - def encode_value(self, prop, value): - return self.converter.encode_prop(prop, value) - - def decode_value(self, prop, value): - return self.converter.decode_prop(prop, value) - - def execute_sql(self, query): - self.cursor.execute(query, None) - self.commit() - - def query_sql(self, query, vars=None): - self.cursor.execute(query, vars) - return self.cursor.fetchall() - - def lookup(self, cls, name, value): - values = [] - qs = 'SELECT * FROM "%s" WHERE ' % self.db_table - found = False - for property in cls.properties(hidden=False): - if property.name == name: - found = True - value = self.encode_value(property, value) - values.append(value) - qs += "%s=" % name - qs += "%s" - if not found: - raise SDBPersistenceError('%s is not a valid field' % name) - qs += ';' - print qs - self.cursor.execute(qs, values) - if self.cursor.rowcount == 1: - row = self.cursor.fetchone() - return self._object_from_row(row, self.cursor.description) - elif self.cursor.rowcount == 0: - raise KeyError, 'Object not found' - else: - raise LookupError, 'Multiple Objects Found' - - def query(self, cls, filters, limit=None, order_by=None): - parts = [] - qs = 'SELECT * FROM "%s"' % self.db_table - if filters: - qs += ' WHERE ' - properties = cls.properties(hidden=False) - for filter, value in filters: - name, op = filter.strip().split() - found = False - for property in properties: - if property.name == name: - found = True - value = self.encode_value(property, value) - parts.append(""""%s"%s'%s'""" % (name, op, value)) - if not found: - raise SDBPersistenceError('%s is not a valid field' % name) - qs += ','.join(parts) - qs += ';' - print qs - cursor = self.connection.cursor() - cursor.execute(qs) - return self._object_lister(cursor) - - def get_property(self, prop, obj, name): - qs = """SELECT "%s" FROM "%s" WHERE id='%s';""" % (name, self.db_table, obj.id) - print qs - self.cursor.execute(qs, None) - if self.cursor.rowcount == 1: - rs = self.cursor.fetchone() - for prop in obj.properties(hidden=False): - if prop.name == name: - v = self.decode_value(prop, rs[0]) - return v - raise AttributeError, '%s not found' % name - - def set_property(self, prop, obj, name, value): - pass - value = self.encode_value(prop, value) - qs = 'UPDATE "%s" SET ' % self.db_table - qs += "%s='%s'" % (name, self.encode_value(prop, value)) - qs += " WHERE id='%s'" % obj.id - qs += ';' - print qs - self.cursor.execute(qs) - self.commit() - - def get_object(self, cls, id): - qs = """SELECT * FROM "%s" WHERE id='%s';""" % (self.db_table, id) - self.cursor.execute(qs, None) - if self.cursor.rowcount == 1: - row = self.cursor.fetchone() - return self._object_from_row(row, self.cursor.description) - else: - raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) - - def get_object_from_id(self, id): - return self.get_object(self.cls, id) - - def _find_calculated_props(self, obj): - return [p for p in obj.properties() if hasattr(p, 'calculated_type')] - - def save_object(self, obj): - obj._auto_update = False - calculated = self._find_calculated_props(obj) - if not obj.id: - obj.id = str(uuid.uuid4()) - qs, values = self._build_insert_qs(obj, calculated) - else: - qs, values = self._build_update_qs(obj, calculated) - print qs - self.cursor.execute(qs, values) - if calculated: - calc_values = self.cursor.fetchone() - print calculated - print calc_values - for i in range(0, len(calculated)): - prop = calculated[i] - prop._set_direct(obj, calc_values[i]) - self.commit() - - def delete_object(self, obj): - qs = """DELETE FROM "%s" WHERE id='%s';""" % (self.db_table, obj.id) - print qs - self.cursor.execute(qs) - self.commit() - - diff --git a/backup/src/boto/sdb/db/manager/sdbmanager.py b/backup/src/boto/sdb/db/manager/sdbmanager.py deleted file mode 100644 index 7dcc1c6..0000000 --- a/backup/src/boto/sdb/db/manager/sdbmanager.py +++ /dev/null @@ -1,660 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -import boto -import re -from boto.utils import find_class -import uuid -from boto.sdb.db.key import Key -from boto.sdb.db.model import Model -from boto.sdb.db.blob import Blob -from boto.sdb.db.property import ListProperty, MapProperty -from datetime import datetime, date, time -from boto.exception import SDBPersistenceError - -ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - -class TimeDecodeError(Exception): - pass - -class SDBConverter: - """ - Responsible for converting base Python types to format compatible with underlying - database. For SimpleDB, that means everything needs to be converted to a string - when stored in SimpleDB and from a string when retrieved. - - To convert a value, pass it to the encode or decode method. The encode method - will take a Python native value and convert to DB format. The decode method will - take a DB format value and convert it to Python native format. To find the appropriate - method to call, the generic encode/decode methods will look for the type-specific - method by searching for a method called "encode_" or "decode_". - """ - def __init__(self, manager): - self.manager = manager - self.type_map = { bool : (self.encode_bool, self.decode_bool), - int : (self.encode_int, self.decode_int), - long : (self.encode_long, self.decode_long), - float : (self.encode_float, self.decode_float), - Model : (self.encode_reference, self.decode_reference), - Key : (self.encode_reference, self.decode_reference), - datetime : (self.encode_datetime, self.decode_datetime), - date : (self.encode_date, self.decode_date), - time : (self.encode_time, self.decode_time), - Blob: (self.encode_blob, self.decode_blob), - } - - def encode(self, item_type, value): - try: - if Model in item_type.mro(): - item_type = Model - except: - pass - if item_type in self.type_map: - encode = self.type_map[item_type][0] - return encode(value) - return value - - def decode(self, item_type, value): - if item_type in self.type_map: - decode = self.type_map[item_type][1] - return decode(value) - return value - - def encode_list(self, prop, value): - if value in (None, []): - return [] - if not isinstance(value, list): - # This is a little trick to avoid encoding when it's just a single value, - # since that most likely means it's from a query - item_type = getattr(prop, "item_type") - return self.encode(item_type, value) - # Just enumerate(value) won't work here because - # we need to add in some zero padding - # We support lists up to 1,000 attributes, since - # SDB technically only supports 1024 attributes anyway - values = {} - for k,v in enumerate(value): - values["%03d" % k] = v - return self.encode_map(prop, values) - - def encode_map(self, prop, value): - if value == None: - return None - if not isinstance(value, dict): - raise ValueError, 'Expected a dict value, got %s' % type(value) - new_value = [] - for key in value: - item_type = getattr(prop, "item_type") - if Model in item_type.mro(): - item_type = Model - encoded_value = self.encode(item_type, value[key]) - if encoded_value != None: - new_value.append('%s:%s' % (key, encoded_value)) - return new_value - - def encode_prop(self, prop, value): - if isinstance(prop, ListProperty): - return self.encode_list(prop, value) - elif isinstance(prop, MapProperty): - return self.encode_map(prop, value) - else: - return self.encode(prop.data_type, value) - - def decode_list(self, prop, value): - if not isinstance(value, list): - value = [value] - if hasattr(prop, 'item_type'): - item_type = getattr(prop, "item_type") - dec_val = {} - for val in value: - if val != None: - k,v = self.decode_map_element(item_type, val) - try: - k = int(k) - except: - k = v - dec_val[k] = v - value = dec_val.values() - return value - - def decode_map(self, prop, value): - if not isinstance(value, list): - value = [value] - ret_value = {} - item_type = getattr(prop, "item_type") - for val in value: - k,v = self.decode_map_element(item_type, val) - ret_value[k] = v - return ret_value - - def decode_map_element(self, item_type, value): - """Decode a single element for a map""" - key = value - if ":" in value: - key, value = value.split(':',1) - if Model in item_type.mro(): - value = item_type(id=value) - else: - value = self.decode(item_type, value) - return (key, value) - - def decode_prop(self, prop, value): - if isinstance(prop, ListProperty): - return self.decode_list(prop, value) - elif isinstance(prop, MapProperty): - return self.decode_map(prop, value) - else: - return self.decode(prop.data_type, value) - - def encode_int(self, value): - value = int(value) - value += 2147483648 - return '%010d' % value - - def decode_int(self, value): - try: - value = int(value) - except: - boto.log.error("Error, %s is not an integer" % value) - value = 0 - value = int(value) - value -= 2147483648 - return int(value) - - def encode_long(self, value): - value = long(value) - value += 9223372036854775808 - return '%020d' % value - - def decode_long(self, value): - value = long(value) - value -= 9223372036854775808 - return value - - def encode_bool(self, value): - if value == True or str(value).lower() in ("true", "yes"): - return 'true' - else: - return 'false' - - def decode_bool(self, value): - if value.lower() == 'true': - return True - else: - return False - - def encode_float(self, value): - """ - See http://tools.ietf.org/html/draft-wood-ldapext-float-00. - """ - s = '%e' % value - l = s.split('e') - mantissa = l[0].ljust(18, '0') - exponent = l[1] - if value == 0.0: - case = '3' - exponent = '000' - elif mantissa[0] != '-' and exponent[0] == '+': - case = '5' - exponent = exponent[1:].rjust(3, '0') - elif mantissa[0] != '-' and exponent[0] == '-': - case = '4' - exponent = 999 + int(exponent) - exponent = '%03d' % exponent - elif mantissa[0] == '-' and exponent[0] == '-': - case = '2' - mantissa = '%f' % (10 + float(mantissa)) - mantissa = mantissa.ljust(18, '0') - exponent = exponent[1:].rjust(3, '0') - else: - case = '1' - mantissa = '%f' % (10 + float(mantissa)) - mantissa = mantissa.ljust(18, '0') - exponent = 999 - int(exponent) - exponent = '%03d' % exponent - return '%s %s %s' % (case, exponent, mantissa) - - def decode_float(self, value): - case = value[0] - exponent = value[2:5] - mantissa = value[6:] - if case == '3': - return 0.0 - elif case == '5': - pass - elif case == '4': - exponent = '%03d' % (int(exponent) - 999) - elif case == '2': - mantissa = '%f' % (float(mantissa) - 10) - exponent = '-' + exponent - else: - mantissa = '%f' % (float(mantissa) - 10) - exponent = '%03d' % abs((int(exponent) - 999)) - return float(mantissa + 'e' + exponent) - - def encode_datetime(self, value): - if isinstance(value, str) or isinstance(value, unicode): - return value - return value.strftime(ISO8601) - - def decode_datetime(self, value): - try: - return datetime.strptime(value, ISO8601) - except: - return None - - def encode_date(self, value): - if isinstance(value, str) or isinstance(value, unicode): - return value - return value.isoformat() - - def decode_date(self, value): - try: - value = value.split("-") - return date(int(value[0]), int(value[1]), int(value[2])) - except: - return None - - encode_time = encode_date - - def decode_time(self, value): - """ converts strings in the form of HH:MM:SS.mmmmmm - (created by datetime.time.isoformat()) to - datetime.time objects. - - Timzone-aware strings ("HH:MM:SS.mmmmmm+HH:MM") won't - be handled right now and will raise TimeDecodeError. - """ - if '-' in value or '+' in value: - # TODO: Handle tzinfo - raise TimeDecodeError("Can't handle timezone aware objects: %r" % value) - tmp = value.split('.') - arg = map(int, tmp[0].split(':')) - if len(tmp) == 2: - arg.append(int(tmp[1])) - return time(*arg) - - def encode_reference(self, value): - if value in (None, 'None', '', ' '): - return None - if isinstance(value, str) or isinstance(value, unicode): - return value - else: - return value.id - - def decode_reference(self, value): - if not value or value == "None": - return None - return value - - def encode_blob(self, value): - if not value: - return None - if isinstance(value, str): - return value - - if not value.id: - bucket = self.manager.get_blob_bucket() - key = bucket.new_key(str(uuid.uuid4())) - value.id = "s3://%s/%s" % (key.bucket.name, key.name) - else: - match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value.id) - if match: - s3 = self.manager.get_s3_connection() - bucket = s3.get_bucket(match.group(1), validate=False) - key = bucket.get_key(match.group(2)) - else: - raise SDBPersistenceError("Invalid Blob ID: %s" % value.id) - - if value.value != None: - key.set_contents_from_string(value.value) - return value.id - - - def decode_blob(self, value): - if not value: - return None - match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value) - if match: - s3 = self.manager.get_s3_connection() - bucket = s3.get_bucket(match.group(1), validate=False) - key = bucket.get_key(match.group(2)) - else: - return None - if key: - return Blob(file=key, id="s3://%s/%s" % (key.bucket.name, key.name)) - else: - return None - -class SDBManager(object): - - def __init__(self, cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, ddl_dir, enable_ssl, consistent=None): - self.cls = cls - self.db_name = db_name - self.db_user = db_user - self.db_passwd = db_passwd - self.db_host = db_host - self.db_port = db_port - self.db_table = db_table - self.ddl_dir = ddl_dir - self.enable_ssl = enable_ssl - self.s3 = None - self.bucket = None - self.converter = SDBConverter(self) - self._sdb = None - self._domain = None - if consistent == None and hasattr(cls, "__consistent__"): - consistent = cls.__consistent__ - self.consistent = consistent - - @property - def sdb(self): - if self._sdb is None: - self._connect() - return self._sdb - - @property - def domain(self): - if self._domain is None: - self._connect() - return self._domain - - def _connect(self): - self._sdb = boto.connect_sdb(aws_access_key_id=self.db_user, - aws_secret_access_key=self.db_passwd, - is_secure=self.enable_ssl) - # This assumes that the domain has already been created - # It's much more efficient to do it this way rather than - # having this make a roundtrip each time to validate. - # The downside is that if the domain doesn't exist, it breaks - self._domain = self._sdb.lookup(self.db_name, validate=False) - if not self._domain: - self._domain = self._sdb.create_domain(self.db_name) - - def _object_lister(self, cls, query_lister): - for item in query_lister: - obj = self.get_object(cls, item.name, item) - if obj: - yield obj - - def encode_value(self, prop, value): - if value == None: - return None - if not prop: - return str(value) - return self.converter.encode_prop(prop, value) - - def decode_value(self, prop, value): - return self.converter.decode_prop(prop, value) - - def get_s3_connection(self): - if not self.s3: - self.s3 = boto.connect_s3(self.db_user, self.db_passwd) - return self.s3 - - def get_blob_bucket(self, bucket_name=None): - s3 = self.get_s3_connection() - bucket_name = "%s-%s" % (s3.aws_access_key_id, self.domain.name) - bucket_name = bucket_name.lower() - try: - self.bucket = s3.get_bucket(bucket_name) - except: - self.bucket = s3.create_bucket(bucket_name) - return self.bucket - - def load_object(self, obj): - if not obj._loaded: - a = self.domain.get_attributes(obj.id,consistent_read=self.consistent) - if a.has_key('__type__'): - for prop in obj.properties(hidden=False): - if a.has_key(prop.name): - value = self.decode_value(prop, a[prop.name]) - value = prop.make_value_from_datastore(value) - try: - setattr(obj, prop.name, value) - except Exception, e: - boto.log.exception(e) - obj._loaded = True - - def get_object(self, cls, id, a=None): - obj = None - if not a: - a = self.domain.get_attributes(id,consistent_read=self.consistent) - if a.has_key('__type__'): - if not cls or a['__type__'] != cls.__name__: - cls = find_class(a['__module__'], a['__type__']) - if cls: - params = {} - for prop in cls.properties(hidden=False): - if a.has_key(prop.name): - value = self.decode_value(prop, a[prop.name]) - value = prop.make_value_from_datastore(value) - params[prop.name] = value - obj = cls(id, **params) - obj._loaded = True - else: - s = '(%s) class %s.%s not found' % (id, a['__module__'], a['__type__']) - boto.log.info('sdbmanager: %s' % s) - return obj - - def get_object_from_id(self, id): - return self.get_object(None, id) - - def query(self, query): - query_str = "select * from `%s` %s" % (self.domain.name, self._build_filter_part(query.model_class, query.filters, query.sort_by, query.select)) - if query.limit: - query_str += " limit %s" % query.limit - rs = self.domain.select(query_str, max_items=query.limit, next_token = query.next_token) - query.rs = rs - return self._object_lister(query.model_class, rs) - - def count(self, cls, filters, quick=True, sort_by=None, select=None): - """ - Get the number of results that would - be returned in this query - """ - query = "select count(*) from `%s` %s" % (self.domain.name, self._build_filter_part(cls, filters, sort_by, select)) - count = 0 - for row in self.domain.select(query): - count += int(row['Count']) - if quick: - return count - return count - - - def _build_filter(self, property, name, op, val): - if name == "__id__": - name = 'itemName()' - if name != "itemName()": - name = '`%s`' % name - if val == None: - if op in ('is','='): - return "%(name)s is null" % {"name": name} - elif op in ('is not', '!='): - return "%s is not null" % name - else: - val = "" - if property.__class__ == ListProperty: - if op in ("is", "="): - op = "like" - elif op in ("!=", "not"): - op = "not like" - if not(op in ["like", "not like"] and val.startswith("%")): - val = "%%:%s" % val - return "%s %s '%s'" % (name, op, val.replace("'", "''")) - - def _build_filter_part(self, cls, filters, order_by=None, select=None): - """ - Build the filter part - """ - import types - query_parts = [] - order_by_filtered = False - if order_by: - if order_by[0] == "-": - order_by_method = "DESC"; - order_by = order_by[1:] - else: - order_by_method = "ASC"; - if isinstance(filters, str) or isinstance(filters, unicode): - query = "WHERE `__type__` = '%s' AND %s" % (cls.__name__, filters) - if order_by != None: - query += " ORDER BY `%s` %s" % (order_by, order_by_method) - return query - - for filter in filters: - filter_parts = [] - filter_props = filter[0] - if type(filter_props) != list: - filter_props = [filter_props] - for filter_prop in filter_props: - (name, op) = filter_prop.strip().split(" ", 1) - value = filter[1] - property = cls.find_property(name) - if name == order_by: - order_by_filtered = True - if types.TypeType(value) == types.ListType: - filter_parts_sub = [] - for val in value: - val = self.encode_value(property, val) - if isinstance(val, list): - for v in val: - filter_parts_sub.append(self._build_filter(property, name, op, v)) - else: - filter_parts_sub.append(self._build_filter(property, name, op, val)) - filter_parts.append("(%s)" % (" OR ".join(filter_parts_sub))) - else: - val = self.encode_value(property, value) - if isinstance(val, list): - for v in val: - filter_parts.append(self._build_filter(property, name, op, v)) - else: - filter_parts.append(self._build_filter(property, name, op, val)) - query_parts.append("(%s)" % (" or ".join(filter_parts))) - - - type_query = "(`__type__` = '%s'" % cls.__name__ - for subclass in self._get_all_decendents(cls).keys(): - type_query += " or `__type__` = '%s'" % subclass - type_query +=")" - query_parts.append(type_query) - - order_by_query = "" - if order_by: - if not order_by_filtered: - query_parts.append("`%s` LIKE '%%'" % order_by) - order_by_query = " ORDER BY `%s` %s" % (order_by, order_by_method) - - if select: - query_parts.append("(%s)" % select) - - if len(query_parts) > 0: - return "WHERE %s %s" % (" AND ".join(query_parts), order_by_query) - else: - return "" - - - def _get_all_decendents(self, cls): - """Get all decendents for a given class""" - decendents = {} - for sc in cls.__sub_classes__: - decendents[sc.__name__] = sc - decendents.update(self._get_all_decendents(sc)) - return decendents - - def query_gql(self, query_string, *args, **kwds): - raise NotImplementedError, "GQL queries not supported in SimpleDB" - - def save_object(self, obj): - if not obj.id: - obj.id = str(uuid.uuid4()) - - attrs = {'__type__' : obj.__class__.__name__, - '__module__' : obj.__class__.__module__, - '__lineage__' : obj.get_lineage()} - del_attrs = [] - for property in obj.properties(hidden=False): - value = property.get_value_for_datastore(obj) - if value is not None: - value = self.encode_value(property, value) - if value == []: - value = None - if value == None: - del_attrs.append(property.name) - continue - attrs[property.name] = value - if property.unique: - try: - args = {property.name: value} - obj2 = obj.find(**args).next() - if obj2.id != obj.id: - raise SDBPersistenceError("Error: %s must be unique!" % property.name) - except(StopIteration): - pass - self.domain.put_attributes(obj.id, attrs, replace=True) - if len(del_attrs) > 0: - self.domain.delete_attributes(obj.id, del_attrs) - return obj - - def delete_object(self, obj): - self.domain.delete_attributes(obj.id) - - def set_property(self, prop, obj, name, value): - value = prop.get_value_for_datastore(obj) - value = self.encode_value(prop, value) - if prop.unique: - try: - args = {prop.name: value} - obj2 = obj.find(**args).next() - if obj2.id != obj.id: - raise SDBPersistenceError("Error: %s must be unique!" % prop.name) - except(StopIteration): - pass - self.domain.put_attributes(obj.id, {name : value}, replace=True) - - def get_property(self, prop, obj, name): - a = self.domain.get_attributes(obj.id,consistent_read=self.consistent) - - # try to get the attribute value from SDB - if name in a: - value = self.decode_value(prop, a[name]) - value = prop.make_value_from_datastore(value) - setattr(obj, prop.name, value) - return value - raise AttributeError, '%s not found' % name - - def set_key_value(self, obj, name, value): - self.domain.put_attributes(obj.id, {name : value}, replace=True) - - def delete_key_value(self, obj, name): - self.domain.delete_attributes(obj.id, name) - - def get_key_value(self, obj, name): - a = self.domain.get_attributes(obj.id, name,consistent_read=self.consistent) - if a.has_key(name): - return a[name] - else: - return None - - def get_raw_item(self, obj): - return self.domain.get_item(obj.id) - diff --git a/backup/src/boto/sdb/db/manager/xmlmanager.py b/backup/src/boto/sdb/db/manager/xmlmanager.py deleted file mode 100644 index 9765df1..0000000 --- a/backup/src/boto/sdb/db/manager/xmlmanager.py +++ /dev/null @@ -1,517 +0,0 @@ -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -import boto -from boto.utils import find_class, Password -from boto.sdb.db.key import Key -from boto.sdb.db.model import Model -from datetime import datetime -from xml.dom.minidom import getDOMImplementation, parse, parseString, Node - -ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - -class XMLConverter: - """ - Responsible for converting base Python types to format compatible with underlying - database. For SimpleDB, that means everything needs to be converted to a string - when stored in SimpleDB and from a string when retrieved. - - To convert a value, pass it to the encode or decode method. The encode method - will take a Python native value and convert to DB format. The decode method will - take a DB format value and convert it to Python native format. To find the appropriate - method to call, the generic encode/decode methods will look for the type-specific - method by searching for a method called "encode_" or "decode_". - """ - def __init__(self, manager): - self.manager = manager - self.type_map = { bool : (self.encode_bool, self.decode_bool), - int : (self.encode_int, self.decode_int), - long : (self.encode_long, self.decode_long), - Model : (self.encode_reference, self.decode_reference), - Key : (self.encode_reference, self.decode_reference), - Password : (self.encode_password, self.decode_password), - datetime : (self.encode_datetime, self.decode_datetime)} - - def get_text_value(self, parent_node): - value = '' - for node in parent_node.childNodes: - if node.nodeType == node.TEXT_NODE: - value += node.data - return value - - def encode(self, item_type, value): - if item_type in self.type_map: - encode = self.type_map[item_type][0] - return encode(value) - return value - - def decode(self, item_type, value): - if item_type in self.type_map: - decode = self.type_map[item_type][1] - return decode(value) - else: - value = self.get_text_value(value) - return value - - def encode_prop(self, prop, value): - if isinstance(value, list): - if hasattr(prop, 'item_type'): - new_value = [] - for v in value: - item_type = getattr(prop, "item_type") - if Model in item_type.mro(): - item_type = Model - new_value.append(self.encode(item_type, v)) - return new_value - else: - return value - else: - return self.encode(prop.data_type, value) - - def decode_prop(self, prop, value): - if prop.data_type == list: - if hasattr(prop, 'item_type'): - item_type = getattr(prop, "item_type") - if Model in item_type.mro(): - item_type = Model - values = [] - for item_node in value.getElementsByTagName('item'): - value = self.decode(item_type, item_node) - values.append(value) - return values - else: - return self.get_text_value(value) - else: - return self.decode(prop.data_type, value) - - def encode_int(self, value): - value = int(value) - return '%d' % value - - def decode_int(self, value): - value = self.get_text_value(value) - if value: - value = int(value) - else: - value = None - return value - - def encode_long(self, value): - value = long(value) - return '%d' % value - - def decode_long(self, value): - value = self.get_text_value(value) - return long(value) - - def encode_bool(self, value): - if value == True: - return 'true' - else: - return 'false' - - def decode_bool(self, value): - value = self.get_text_value(value) - if value.lower() == 'true': - return True - else: - return False - - def encode_datetime(self, value): - return value.strftime(ISO8601) - - def decode_datetime(self, value): - value = self.get_text_value(value) - try: - return datetime.strptime(value, ISO8601) - except: - return None - - def encode_reference(self, value): - if isinstance(value, str) or isinstance(value, unicode): - return value - if value == None: - return '' - else: - val_node = self.manager.doc.createElement("object") - val_node.setAttribute('id', value.id) - val_node.setAttribute('class', '%s.%s' % (value.__class__.__module__, value.__class__.__name__)) - return val_node - - def decode_reference(self, value): - if not value: - return None - try: - value = value.childNodes[0] - class_name = value.getAttribute("class") - id = value.getAttribute("id") - cls = find_class(class_name) - return cls.get_by_ids(id) - except: - return None - - def encode_password(self, value): - if value and len(value) > 0: - return str(value) - else: - return None - - def decode_password(self, value): - value = self.get_text_value(value) - return Password(value) - - -class XMLManager(object): - - def __init__(self, cls, db_name, db_user, db_passwd, - db_host, db_port, db_table, ddl_dir, enable_ssl): - self.cls = cls - if not db_name: - db_name = cls.__name__.lower() - self.db_name = db_name - self.db_user = db_user - self.db_passwd = db_passwd - self.db_host = db_host - self.db_port = db_port - self.db_table = db_table - self.ddl_dir = ddl_dir - self.s3 = None - self.converter = XMLConverter(self) - self.impl = getDOMImplementation() - self.doc = self.impl.createDocument(None, 'objects', None) - - self.connection = None - self.enable_ssl = enable_ssl - self.auth_header = None - if self.db_user: - import base64 - base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1] - authheader = "Basic %s" % base64string - self.auth_header = authheader - - def _connect(self): - if self.db_host: - if self.enable_ssl: - from httplib import HTTPSConnection as Connection - else: - from httplib import HTTPConnection as Connection - - self.connection = Connection(self.db_host, self.db_port) - - def _make_request(self, method, url, post_data=None, body=None): - """ - Make a request on this connection - """ - if not self.connection: - self._connect() - try: - self.connection.close() - except: - pass - self.connection.connect() - headers = {} - if self.auth_header: - headers["Authorization"] = self.auth_header - self.connection.request(method, url, body, headers) - resp = self.connection.getresponse() - return resp - - def new_doc(self): - return self.impl.createDocument(None, 'objects', None) - - def _object_lister(self, cls, doc): - for obj_node in doc.getElementsByTagName('object'): - if not cls: - class_name = obj_node.getAttribute('class') - cls = find_class(class_name) - id = obj_node.getAttribute('id') - obj = cls(id) - for prop_node in obj_node.getElementsByTagName('property'): - prop_name = prop_node.getAttribute('name') - prop = obj.find_property(prop_name) - if prop: - if hasattr(prop, 'item_type'): - value = self.get_list(prop_node, prop.item_type) - else: - value = self.decode_value(prop, prop_node) - value = prop.make_value_from_datastore(value) - setattr(obj, prop.name, value) - yield obj - - def reset(self): - self._connect() - - def get_doc(self): - return self.doc - - def encode_value(self, prop, value): - return self.converter.encode_prop(prop, value) - - def decode_value(self, prop, value): - return self.converter.decode_prop(prop, value) - - def get_s3_connection(self): - if not self.s3: - self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) - return self.s3 - - def get_list(self, prop_node, item_type): - values = [] - try: - items_node = prop_node.getElementsByTagName('items')[0] - except: - return [] - for item_node in items_node.getElementsByTagName('item'): - value = self.converter.decode(item_type, item_node) - values.append(value) - return values - - def get_object_from_doc(self, cls, id, doc): - obj_node = doc.getElementsByTagName('object')[0] - if not cls: - class_name = obj_node.getAttribute('class') - cls = find_class(class_name) - if not id: - id = obj_node.getAttribute('id') - obj = cls(id) - for prop_node in obj_node.getElementsByTagName('property'): - prop_name = prop_node.getAttribute('name') - prop = obj.find_property(prop_name) - value = self.decode_value(prop, prop_node) - value = prop.make_value_from_datastore(value) - if value != None: - try: - setattr(obj, prop.name, value) - except: - pass - return obj - - def get_props_from_doc(self, cls, id, doc): - """ - Pull out the properties from this document - Returns the class, the properties in a hash, and the id if provided as a tuple - :return: (cls, props, id) - """ - obj_node = doc.getElementsByTagName('object')[0] - if not cls: - class_name = obj_node.getAttribute('class') - cls = find_class(class_name) - if not id: - id = obj_node.getAttribute('id') - props = {} - for prop_node in obj_node.getElementsByTagName('property'): - prop_name = prop_node.getAttribute('name') - prop = cls.find_property(prop_name) - value = self.decode_value(prop, prop_node) - value = prop.make_value_from_datastore(value) - if value != None: - props[prop.name] = value - return (cls, props, id) - - - def get_object(self, cls, id): - if not self.connection: - self._connect() - - if not self.connection: - raise NotImplementedError("Can't query without a database connection") - url = "/%s/%s" % (self.db_name, id) - resp = self._make_request('GET', url) - if resp.status == 200: - doc = parse(resp) - else: - raise Exception("Error: %s" % resp.status) - return self.get_object_from_doc(cls, id, doc) - - def query(self, cls, filters, limit=None, order_by=None): - if not self.connection: - self._connect() - - if not self.connection: - raise NotImplementedError("Can't query without a database connection") - - from urllib import urlencode - - query = str(self._build_query(cls, filters, limit, order_by)) - if query: - url = "/%s?%s" % (self.db_name, urlencode({"query": query})) - else: - url = "/%s" % self.db_name - resp = self._make_request('GET', url) - if resp.status == 200: - doc = parse(resp) - else: - raise Exception("Error: %s" % resp.status) - return self._object_lister(cls, doc) - - def _build_query(self, cls, filters, limit, order_by): - import types - if len(filters) > 4: - raise Exception('Too many filters, max is 4') - parts = [] - properties = cls.properties(hidden=False) - for filter, value in filters: - name, op = filter.strip().split() - found = False - for property in properties: - if property.name == name: - found = True - if types.TypeType(value) == types.ListType: - filter_parts = [] - for val in value: - val = self.encode_value(property, val) - filter_parts.append("'%s' %s '%s'" % (name, op, val)) - parts.append("[%s]" % " OR ".join(filter_parts)) - else: - value = self.encode_value(property, value) - parts.append("['%s' %s '%s']" % (name, op, value)) - if not found: - raise Exception('%s is not a valid field' % name) - if order_by: - if order_by.startswith("-"): - key = order_by[1:] - type = "desc" - else: - key = order_by - type = "asc" - parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type)) - return ' intersection '.join(parts) - - def query_gql(self, query_string, *args, **kwds): - raise NotImplementedError, "GQL queries not supported in XML" - - def save_list(self, doc, items, prop_node): - items_node = doc.createElement('items') - prop_node.appendChild(items_node) - for item in items: - item_node = doc.createElement('item') - items_node.appendChild(item_node) - if isinstance(item, Node): - item_node.appendChild(item) - else: - text_node = doc.createTextNode(item) - item_node.appendChild(text_node) - - def save_object(self, obj): - """ - Marshal the object and do a PUT - """ - doc = self.marshal_object(obj) - if obj.id: - url = "/%s/%s" % (self.db_name, obj.id) - else: - url = "/%s" % (self.db_name) - resp = self._make_request("PUT", url, body=doc.toxml()) - new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp)) - obj.id = new_obj.id - for prop in obj.properties(): - try: - propname = prop.name - except AttributeError: - propname = None - if propname: - value = getattr(new_obj, prop.name) - if value: - setattr(obj, prop.name, value) - return obj - - - def marshal_object(self, obj, doc=None): - if not doc: - doc = self.new_doc() - if not doc: - doc = self.doc - obj_node = doc.createElement('object') - - if obj.id: - obj_node.setAttribute('id', obj.id) - - obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__, - obj.__class__.__name__)) - root = doc.documentElement - root.appendChild(obj_node) - for property in obj.properties(hidden=False): - prop_node = doc.createElement('property') - prop_node.setAttribute('name', property.name) - prop_node.setAttribute('type', property.type_name) - value = property.get_value_for_datastore(obj) - if value is not None: - value = self.encode_value(property, value) - if isinstance(value, list): - self.save_list(doc, value, prop_node) - elif isinstance(value, Node): - prop_node.appendChild(value) - else: - text_node = doc.createTextNode(unicode(value).encode("ascii", "ignore")) - prop_node.appendChild(text_node) - obj_node.appendChild(prop_node) - - return doc - - def unmarshal_object(self, fp, cls=None, id=None): - if isinstance(fp, str) or isinstance(fp, unicode): - doc = parseString(fp) - else: - doc = parse(fp) - return self.get_object_from_doc(cls, id, doc) - - def unmarshal_props(self, fp, cls=None, id=None): - """ - Same as unmarshalling an object, except it returns - from "get_props_from_doc" - """ - if isinstance(fp, str) or isinstance(fp, unicode): - doc = parseString(fp) - else: - doc = parse(fp) - return self.get_props_from_doc(cls, id, doc) - - def delete_object(self, obj): - url = "/%s/%s" % (self.db_name, obj.id) - return self._make_request("DELETE", url) - - def set_key_value(self, obj, name, value): - self.domain.put_attributes(obj.id, {name : value}, replace=True) - - def delete_key_value(self, obj, name): - self.domain.delete_attributes(obj.id, name) - - def get_key_value(self, obj, name): - a = self.domain.get_attributes(obj.id, name) - if a.has_key(name): - return a[name] - else: - return None - - def get_raw_item(self, obj): - return self.domain.get_item(obj.id) - - def set_property(self, prop, obj, name, value): - pass - - def get_property(self, prop, obj, name): - pass - - def load_object(self, obj): - if not obj._loaded: - obj = obj.get_by_id(obj.id) - obj._loaded = True - return obj - diff --git a/backup/src/boto/sdb/db/model.py b/backup/src/boto/sdb/db/model.py deleted file mode 100644 index 18bec4b..0000000 --- a/backup/src/boto/sdb/db/model.py +++ /dev/null @@ -1,248 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.sdb.db.manager import get_manager -from boto.sdb.db.property import Property -from boto.sdb.db.key import Key -from boto.sdb.db.query import Query -import boto - -class ModelMeta(type): - "Metaclass for all Models" - - def __init__(cls, name, bases, dict): - super(ModelMeta, cls).__init__(name, bases, dict) - # Make sure this is a subclass of Model - mainly copied from django ModelBase (thanks!) - cls.__sub_classes__ = [] - try: - if filter(lambda b: issubclass(b, Model), bases): - for base in bases: - base.__sub_classes__.append(cls) - cls._manager = get_manager(cls) - # look for all of the Properties and set their names - for key in dict.keys(): - if isinstance(dict[key], Property): - property = dict[key] - property.__property_config__(cls, key) - prop_names = [] - props = cls.properties() - for prop in props: - if not prop.__class__.__name__.startswith('_'): - prop_names.append(prop.name) - setattr(cls, '_prop_names', prop_names) - except NameError: - # 'Model' isn't defined yet, meaning we're looking at our own - # Model class, defined below. - pass - -class Model(object): - __metaclass__ = ModelMeta - __consistent__ = False # Consistent is set off by default - id = None - - @classmethod - def get_lineage(cls): - l = [c.__name__ for c in cls.mro()] - l.reverse() - return '.'.join(l) - - @classmethod - def kind(cls): - return cls.__name__ - - @classmethod - def _get_by_id(cls, id, manager=None): - if not manager: - manager = cls._manager - return manager.get_object(cls, id) - - @classmethod - def get_by_id(cls, ids=None, parent=None): - if isinstance(ids, list): - objs = [cls._get_by_id(id) for id in ids] - return objs - else: - return cls._get_by_id(ids) - - get_by_ids = get_by_id - - @classmethod - def get_by_key_name(cls, key_names, parent=None): - raise NotImplementedError, "Key Names are not currently supported" - - @classmethod - def find(cls, limit=None, next_token=None, **params): - q = Query(cls, limit=limit, next_token=next_token) - for key, value in params.items(): - q.filter('%s =' % key, value) - return q - - @classmethod - def all(cls, limit=None, next_token=None): - return cls.find(limit=limit, next_token=next_token) - - @classmethod - def get_or_insert(key_name, **kw): - raise NotImplementedError, "get_or_insert not currently supported" - - @classmethod - def properties(cls, hidden=True): - properties = [] - while cls: - for key in cls.__dict__.keys(): - prop = cls.__dict__[key] - if isinstance(prop, Property): - if hidden or not prop.__class__.__name__.startswith('_'): - properties.append(prop) - if len(cls.__bases__) > 0: - cls = cls.__bases__[0] - else: - cls = None - return properties - - @classmethod - def find_property(cls, prop_name): - property = None - while cls: - for key in cls.__dict__.keys(): - prop = cls.__dict__[key] - if isinstance(prop, Property): - if not prop.__class__.__name__.startswith('_') and prop_name == prop.name: - property = prop - if len(cls.__bases__) > 0: - cls = cls.__bases__[0] - else: - cls = None - return property - - @classmethod - def get_xmlmanager(cls): - if not hasattr(cls, '_xmlmanager'): - from boto.sdb.db.manager.xmlmanager import XMLManager - cls._xmlmanager = XMLManager(cls, None, None, None, - None, None, None, None, False) - return cls._xmlmanager - - @classmethod - def from_xml(cls, fp): - xmlmanager = cls.get_xmlmanager() - return xmlmanager.unmarshal_object(fp) - - def __init__(self, id=None, **kw): - self._loaded = False - # first try to initialize all properties to their default values - for prop in self.properties(hidden=False): - try: - setattr(self, prop.name, prop.default_value()) - except ValueError: - pass - if kw.has_key('manager'): - self._manager = kw['manager'] - self.id = id - for key in kw: - if key != 'manager': - # We don't want any errors populating up when loading an object, - # so if it fails we just revert to it's default value - try: - setattr(self, key, kw[key]) - except Exception, e: - boto.log.exception(e) - - def __repr__(self): - return '%s<%s>' % (self.__class__.__name__, self.id) - - def __str__(self): - return str(self.id) - - def __eq__(self, other): - return other and isinstance(other, Model) and self.id == other.id - - def _get_raw_item(self): - return self._manager.get_raw_item(self) - - def load(self): - if self.id and not self._loaded: - self._manager.load_object(self) - - def reload(self): - if self.id: - self._loaded = False - self._manager.load_object(self) - - def put(self): - self._manager.save_object(self) - - save = put - - def delete(self): - self._manager.delete_object(self) - - def key(self): - return Key(obj=self) - - def set_manager(self, manager): - self._manager = manager - - def to_dict(self): - props = {} - for prop in self.properties(hidden=False): - props[prop.name] = getattr(self, prop.name) - obj = {'properties' : props, - 'id' : self.id} - return {self.__class__.__name__ : obj} - - def to_xml(self, doc=None): - xmlmanager = self.get_xmlmanager() - doc = xmlmanager.marshal_object(self, doc) - return doc - - @classmethod - def find_subclass(cls, name): - """Find a subclass with a given name""" - if name == cls.__name__: - return cls - for sc in cls.__sub_classes__: - r = sc.find_subclass(name) - if r != None: - return r - -class Expando(Model): - - def __setattr__(self, name, value): - if name in self._prop_names: - object.__setattr__(self, name, value) - elif name.startswith('_'): - object.__setattr__(self, name, value) - elif name == 'id': - object.__setattr__(self, name, value) - else: - self._manager.set_key_value(self, name, value) - object.__setattr__(self, name, value) - - def __getattr__(self, name): - if not name.startswith('_'): - value = self._manager.get_key_value(self, name) - if value: - object.__setattr__(self, name, value) - return value - raise AttributeError - - diff --git a/backup/src/boto/sdb/db/property.py b/backup/src/boto/sdb/db/property.py deleted file mode 100644 index 6db7cac..0000000 --- a/backup/src/boto/sdb/db/property.py +++ /dev/null @@ -1,624 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import datetime -from key import Key -from boto.utils import Password -from boto.sdb.db.query import Query -import re -import boto -import boto.s3.key -from boto.sdb.db.blob import Blob - -class Property(object): - - data_type = str - type_name = '' - name = '' - verbose_name = '' - - def __init__(self, verbose_name=None, name=None, default=None, required=False, - validator=None, choices=None, unique=False): - self.verbose_name = verbose_name - self.name = name - self.default = default - self.required = required - self.validator = validator - self.choices = choices - if self.name: - self.slot_name = '_' + self.name - else: - self.slot_name = '_' - self.unique = unique - - def __get__(self, obj, objtype): - if obj: - obj.load() - return getattr(obj, self.slot_name) - else: - return None - - def __set__(self, obj, value): - self.validate(value) - - # Fire off any on_set functions - try: - if obj._loaded and hasattr(obj, "on_set_%s" % self.name): - fnc = getattr(obj, "on_set_%s" % self.name) - value = fnc(value) - except Exception: - boto.log.exception("Exception running on_set_%s" % self.name) - - setattr(obj, self.slot_name, value) - - def __property_config__(self, model_class, property_name): - self.model_class = model_class - self.name = property_name - self.slot_name = '_' + self.name - - def default_validator(self, value): - if value == self.default_value(): - return - if not isinstance(value, self.data_type): - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) - - def default_value(self): - return self.default - - def validate(self, value): - if self.required and value==None: - raise ValueError, '%s is a required property' % self.name - if self.choices and value and not value in self.choices: - raise ValueError, '%s not a valid choice for %s.%s' % (value, self.model_class.__name__, self.name) - if self.validator: - self.validator(value) - else: - self.default_validator(value) - return value - - def empty(self, value): - return not value - - def get_value_for_datastore(self, model_instance): - return getattr(model_instance, self.name) - - def make_value_from_datastore(self, value): - return value - - def get_choices(self): - if callable(self.choices): - return self.choices() - return self.choices - -def validate_string(value): - if value == None: - return - elif isinstance(value, str) or isinstance(value, unicode): - if len(value) > 1024: - raise ValueError, 'Length of value greater than maxlength' - else: - raise TypeError, 'Expecting String, got %s' % type(value) - -class StringProperty(Property): - - type_name = 'String' - - def __init__(self, verbose_name=None, name=None, default='', required=False, - validator=validate_string, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - -class TextProperty(Property): - - type_name = 'Text' - - def __init__(self, verbose_name=None, name=None, default='', required=False, - validator=None, choices=None, unique=False, max_length=None): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - self.max_length = max_length - - def validate(self, value): - if not isinstance(value, str) and not isinstance(value, unicode): - raise TypeError, 'Expecting Text, got %s' % type(value) - if self.max_length and len(value) > self.max_length: - raise ValueError, 'Length of value greater than maxlength %s' % self.max_length - -class PasswordProperty(StringProperty): - """ - Hashed property who's original value can not be - retrieved, but still can be compaired. - """ - data_type = Password - type_name = 'Password' - - def __init__(self, verbose_name=None, name=None, default='', required=False, - validator=None, choices=None, unique=False): - StringProperty.__init__(self, verbose_name, name, default, required, validator, choices, unique) - - def make_value_from_datastore(self, value): - p = Password(value) - return p - - def get_value_for_datastore(self, model_instance): - value = StringProperty.get_value_for_datastore(self, model_instance) - if value and len(value): - return str(value) - else: - return None - - def __set__(self, obj, value): - if not isinstance(value, Password): - p = Password() - p.set(value) - value = p - Property.__set__(self, obj, value) - - def __get__(self, obj, objtype): - return Password(StringProperty.__get__(self, obj, objtype)) - - def validate(self, value): - value = Property.validate(self, value) - if isinstance(value, Password): - if len(value) > 1024: - raise ValueError, 'Length of value greater than maxlength' - else: - raise TypeError, 'Expecting Password, got %s' % type(value) - -class BlobProperty(Property): - data_type = Blob - type_name = "blob" - - def __set__(self, obj, value): - if value != self.default_value(): - if not isinstance(value, Blob): - oldb = self.__get__(obj, type(obj)) - id = None - if oldb: - id = oldb.id - b = Blob(value=value, id=id) - value = b - Property.__set__(self, obj, value) - -class S3KeyProperty(Property): - - data_type = boto.s3.key.Key - type_name = 'S3Key' - validate_regex = "^s3:\/\/([^\/]*)\/(.*)$" - - def __init__(self, verbose_name=None, name=None, default=None, - required=False, validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, - validator, choices, unique) - - def validate(self, value): - if value == self.default_value() or value == str(self.default_value()): - return self.default_value() - if isinstance(value, self.data_type): - return - match = re.match(self.validate_regex, value) - if match: - return - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) - - def __get__(self, obj, objtype): - value = Property.__get__(self, obj, objtype) - if value: - if isinstance(value, self.data_type): - return value - match = re.match(self.validate_regex, value) - if match: - s3 = obj._manager.get_s3_connection() - bucket = s3.get_bucket(match.group(1), validate=False) - k = bucket.get_key(match.group(2)) - if not k: - k = bucket.new_key(match.group(2)) - k.set_contents_from_string("") - return k - else: - return value - - def get_value_for_datastore(self, model_instance): - value = Property.get_value_for_datastore(self, model_instance) - if value: - return "s3://%s/%s" % (value.bucket.name, value.name) - else: - return None - -class IntegerProperty(Property): - - data_type = int - type_name = 'Integer' - - def __init__(self, verbose_name=None, name=None, default=0, required=False, - validator=None, choices=None, unique=False, max=2147483647, min=-2147483648): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - self.max = max - self.min = min - - def validate(self, value): - value = int(value) - value = Property.validate(self, value) - if value > self.max: - raise ValueError, 'Maximum value is %d' % self.max - if value < self.min: - raise ValueError, 'Minimum value is %d' % self.min - return value - - def empty(self, value): - return value is None - - def __set__(self, obj, value): - if value == "" or value == None: - value = 0 - return Property.__set__(self, obj, value) - - - -class LongProperty(Property): - - data_type = long - type_name = 'Long' - - def __init__(self, verbose_name=None, name=None, default=0, required=False, - validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - - def validate(self, value): - value = long(value) - value = Property.validate(self, value) - min = -9223372036854775808 - max = 9223372036854775807 - if value > max: - raise ValueError, 'Maximum value is %d' % max - if value < min: - raise ValueError, 'Minimum value is %d' % min - return value - - def empty(self, value): - return value is None - -class BooleanProperty(Property): - - data_type = bool - type_name = 'Boolean' - - def __init__(self, verbose_name=None, name=None, default=False, required=False, - validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - - def empty(self, value): - return value is None - -class FloatProperty(Property): - - data_type = float - type_name = 'Float' - - def __init__(self, verbose_name=None, name=None, default=0.0, required=False, - validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - - def validate(self, value): - value = float(value) - value = Property.validate(self, value) - return value - - def empty(self, value): - return value is None - -class DateTimeProperty(Property): - - data_type = datetime.datetime - type_name = 'DateTime' - - def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, - default=None, required=False, validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - self.auto_now = auto_now - self.auto_now_add = auto_now_add - - def default_value(self): - if self.auto_now or self.auto_now_add: - return self.now() - return Property.default_value(self) - - def validate(self, value): - if value == None: - return - if not isinstance(value, self.data_type): - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) - - def get_value_for_datastore(self, model_instance): - if self.auto_now: - setattr(model_instance, self.name, self.now()) - return Property.get_value_for_datastore(self, model_instance) - - def now(self): - return datetime.datetime.utcnow() - -class DateProperty(Property): - - data_type = datetime.date - type_name = 'Date' - - def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, - default=None, required=False, validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - self.auto_now = auto_now - self.auto_now_add = auto_now_add - - def default_value(self): - if self.auto_now or self.auto_now_add: - return self.now() - return Property.default_value(self) - - def validate(self, value): - if value == None: - return - if not isinstance(value, self.data_type): - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) - - def get_value_for_datastore(self, model_instance): - if self.auto_now: - setattr(model_instance, self.name, self.now()) - val = Property.get_value_for_datastore(self, model_instance) - if isinstance(val, datetime.datetime): - val = val.date() - return val - - def now(self): - return datetime.date.today() - - -class TimeProperty(Property): - data_type = datetime.time - type_name = 'Time' - - def __init__(self, verbose_name=None, name=None, - default=None, required=False, validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - - def validate(self, value): - if value is None: - return - if not isinstance(value, self.data_type): - raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) - - -class ReferenceProperty(Property): - - data_type = Key - type_name = 'Reference' - - def __init__(self, reference_class=None, collection_name=None, - verbose_name=None, name=None, default=None, required=False, validator=None, choices=None, unique=False): - Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) - self.reference_class = reference_class - self.collection_name = collection_name - - def __get__(self, obj, objtype): - if obj: - value = getattr(obj, self.slot_name) - if value == self.default_value(): - return value - # If the value is still the UUID for the referenced object, we need to create - # the object now that is the attribute has actually been accessed. This lazy - # instantiation saves unnecessary roundtrips to SimpleDB - if isinstance(value, str) or isinstance(value, unicode): - value = self.reference_class(value) - setattr(obj, self.name, value) - return value - - def __set__(self, obj, value): - """Don't allow this object to be associated to itself - This causes bad things to happen""" - if value != None and (obj.id == value or (hasattr(value, "id") and obj.id == value.id)): - raise ValueError, "Can not associate an object with itself!" - return super(ReferenceProperty, self).__set__(obj,value) - - def __property_config__(self, model_class, property_name): - Property.__property_config__(self, model_class, property_name) - if self.collection_name is None: - self.collection_name = '%s_%s_set' % (model_class.__name__.lower(), self.name) - if hasattr(self.reference_class, self.collection_name): - raise ValueError, 'duplicate property: %s' % self.collection_name - setattr(self.reference_class, self.collection_name, - _ReverseReferenceProperty(model_class, property_name, self.collection_name)) - - def check_uuid(self, value): - # This does a bit of hand waving to "type check" the string - t = value.split('-') - if len(t) != 5: - raise ValueError - - def check_instance(self, value): - try: - obj_lineage = value.get_lineage() - cls_lineage = self.reference_class.get_lineage() - if obj_lineage.startswith(cls_lineage): - return - raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) - except: - raise ValueError, '%s is not a Model' % value - - def validate(self, value): - if self.required and value==None: - raise ValueError, '%s is a required property' % self.name - if value == self.default_value(): - return - if not isinstance(value, str) and not isinstance(value, unicode): - self.check_instance(value) - -class _ReverseReferenceProperty(Property): - data_type = Query - type_name = 'query' - - def __init__(self, model, prop, name): - self.__model = model - self.__property = prop - self.collection_name = prop - self.name = name - self.item_type = model - - def __get__(self, model_instance, model_class): - """Fetches collection of model instances of this collection property.""" - if model_instance is not None: - query = Query(self.__model) - if type(self.__property) == list: - props = [] - for prop in self.__property: - props.append("%s =" % prop) - return query.filter(props, model_instance) - else: - return query.filter(self.__property + ' =', model_instance) - else: - return self - - def __set__(self, model_instance, value): - """Not possible to set a new collection.""" - raise ValueError, 'Virtual property is read-only' - - -class CalculatedProperty(Property): - - def __init__(self, verbose_name=None, name=None, default=None, - required=False, validator=None, choices=None, - calculated_type=int, unique=False, use_method=False): - Property.__init__(self, verbose_name, name, default, required, - validator, choices, unique) - self.calculated_type = calculated_type - self.use_method = use_method - - def __get__(self, obj, objtype): - value = self.default_value() - if obj: - try: - value = getattr(obj, self.slot_name) - if self.use_method: - value = value() - except AttributeError: - pass - return value - - def __set__(self, obj, value): - """Not possible to set a new AutoID.""" - pass - - def _set_direct(self, obj, value): - if not self.use_method: - setattr(obj, self.slot_name, value) - - def get_value_for_datastore(self, model_instance): - if self.calculated_type in [str, int, bool]: - value = self.__get__(model_instance, model_instance.__class__) - return value - else: - return None - -class ListProperty(Property): - - data_type = list - type_name = 'List' - - def __init__(self, item_type, verbose_name=None, name=None, default=None, **kwds): - if default is None: - default = [] - self.item_type = item_type - Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) - - def validate(self, value): - if value is not None: - if not isinstance(value, list): - value = [value] - - if self.item_type in (int, long): - item_type = (int, long) - elif self.item_type in (str, unicode): - item_type = (str, unicode) - else: - item_type = self.item_type - - for item in value: - if not isinstance(item, item_type): - if item_type == (int, long): - raise ValueError, 'Items in the %s list must all be integers.' % self.name - else: - raise ValueError('Items in the %s list must all be %s instances' % - (self.name, self.item_type.__name__)) - return value - - def empty(self, value): - return value is None - - def default_value(self): - return list(super(ListProperty, self).default_value()) - - def __set__(self, obj, value): - """Override the set method to allow them to set the property to an instance of the item_type instead of requiring a list to be passed in""" - if self.item_type in (int, long): - item_type = (int, long) - elif self.item_type in (str, unicode): - item_type = (str, unicode) - else: - item_type = self.item_type - if isinstance(value, item_type): - value = [value] - elif value == None: # Override to allow them to set this to "None" to remove everything - value = [] - return super(ListProperty, self).__set__(obj,value) - - -class MapProperty(Property): - - data_type = dict - type_name = 'Map' - - def __init__(self, item_type=str, verbose_name=None, name=None, default=None, **kwds): - if default is None: - default = {} - self.item_type = item_type - Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) - - def validate(self, value): - if value is not None: - if not isinstance(value, dict): - raise ValueError, 'Value must of type dict' - - if self.item_type in (int, long): - item_type = (int, long) - elif self.item_type in (str, unicode): - item_type = (str, unicode) - else: - item_type = self.item_type - - for key in value: - if not isinstance(value[key], item_type): - if item_type == (int, long): - raise ValueError, 'Values in the %s Map must all be integers.' % self.name - else: - raise ValueError('Values in the %s Map must all be %s instances' % - (self.name, self.item_type.__name__)) - return value - - def empty(self, value): - return value is None - - def default_value(self): - return {} diff --git a/backup/src/boto/sdb/db/query.py b/backup/src/boto/sdb/db/query.py deleted file mode 100644 index 31b71aa..0000000 --- a/backup/src/boto/sdb/db/query.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -class Query(object): - __local_iter__ = None - def __init__(self, model_class, limit=None, next_token=None, manager=None): - self.model_class = model_class - self.limit = limit - self.offset = 0 - if manager: - self.manager = manager - else: - self.manager = self.model_class._manager - self.filters = [] - self.select = None - self.sort_by = None - self.rs = None - self.next_token = next_token - - def __iter__(self): - return iter(self.manager.query(self)) - - def next(self): - if self.__local_iter__ == None: - self.__local_iter__ = self.__iter__() - return self.__local_iter__.next() - - def filter(self, property_operator, value): - self.filters.append((property_operator, value)) - return self - - def fetch(self, limit, offset=0): - """Not currently fully supported, but we can use this - to allow them to set a limit in a chainable method""" - self.limit = limit - self.offset = offset - return self - - def count(self, quick=True): - return self.manager.count(self.model_class, self.filters, quick, self.sort_by, self.select) - - def get_query(self): - return self.manager._build_filter_part(self.model_class, self.filters, self.sort_by, self.select) - - def order(self, key): - self.sort_by = key - return self - - def to_xml(self, doc=None): - if not doc: - xmlmanager = self.model_class.get_xmlmanager() - doc = xmlmanager.new_doc() - for obj in self: - obj.to_xml(doc) - return doc - - def get_next_token(self): - if self.rs: - return self.rs.next_token - if self._next_token: - return self._next_token - return None - - def set_next_token(self, token): - self._next_token = token - - next_token = property(get_next_token, set_next_token) diff --git a/backup/src/boto/sdb/db/sequence.py b/backup/src/boto/sdb/db/sequence.py deleted file mode 100644 index 5d7dc10..0000000 --- a/backup/src/boto/sdb/db/sequence.py +++ /dev/null @@ -1,221 +0,0 @@ -# Copyright (c) 2010 Chris Moyer http://coredumped.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.exception import SDBResponseError - -class SequenceGenerator(object): - """Generic Sequence Generator object, this takes a single - string as the "sequence" and uses that to figure out - what the next value in a string is. For example - if you give "ABC" and pass in "A" it will give you "B", - and if you give it "C" it will give you "AA". - - If you set "rollover" to True in the above example, passing - in "C" would give you "A" again. - - The Sequence string can be a string or any iterable - that has the "index" function and is indexable. - """ - __name__ = "SequenceGenerator" - - def __init__(self, sequence_string, rollover=False): - """Create a new SequenceGenerator using the sequence_string - as how to generate the next item. - - :param sequence_string: The string or list that explains - how to generate the next item in the sequence - :type sequence_string: str,iterable - - :param rollover: Rollover instead of incrementing when - we hit the end of the sequence - :type rollover: bool - """ - self.sequence_string = sequence_string - self.sequence_length = len(sequence_string[0]) - self.rollover = rollover - self.last_item = sequence_string[-1] - self.__name__ = "%s('%s')" % (self.__class__.__name__, sequence_string) - - def __call__(self, val, last=None): - """Get the next value in the sequence""" - # If they pass us in a string that's not at least - # the lenght of our sequence, then return the - # first element in our sequence - if val == None or len(val) < self.sequence_length: - return self.sequence_string[0] - last_value = val[-self.sequence_length:] - if (not self.rollover) and (last_value == self.last_item): - val = "%s%s" % (self(val[:-self.sequence_length]), self._inc(last_value)) - else: - val = "%s%s" % (val[:-self.sequence_length], self._inc(last_value)) - return val - - def _inc(self, val): - """Increment a single value""" - assert(len(val) == self.sequence_length) - return self.sequence_string[(self.sequence_string.index(val)+1) % len(self.sequence_string)] - - - -# -# Simple Sequence Functions -# -def increment_by_one(cv=None, lv=None): - if cv == None: - return 0 - return cv + 1 - -def double(cv=None, lv=None): - if cv == None: - return 1 - return cv * 2 - -def fib(cv=1, lv=0): - """The fibonacci sequence, this incrementer uses the - last value""" - if cv == None: - cv = 1 - if lv == None: - lv = 0 - return cv + lv - -increment_string = SequenceGenerator("ABCDEFGHIJKLMNOPQRSTUVWXYZ") - - - -class Sequence(object): - """A simple Sequence using the new SDB "Consistent" features - Based largly off of the "Counter" example from mitch garnaat: - http://bitbucket.org/mitch/stupidbototricks/src/tip/counter.py""" - - - def __init__(self, id=None, domain_name=None, fnc=increment_by_one, init_val=None): - """Create a new Sequence, using an optional function to - increment to the next number, by default we just increment by one. - Every parameter here is optional, if you don't specify any options - then you'll get a new SequenceGenerator with a random ID stored in the - default domain that increments by one and uses the default botoweb - environment - - :param id: Optional ID (name) for this counter - :type id: str - - :param domain_name: Optional domain name to use, by default we get this out of the - environment configuration - :type domain_name:str - - :param fnc: Optional function to use for the incrementation, by default we just increment by one - There are several functions defined in this module. - Your function must accept "None" to get the initial value - :type fnc: function, str - - :param init_val: Initial value, by default this is the first element in your sequence, - but you can pass in any value, even a string if you pass in a function that uses - strings instead of ints to increment - """ - self._db = None - self._value = None - self.last_value = None - self.domain_name = domain_name - self.id = id - if self.id == None: - import uuid - self.id = str(uuid.uuid4()) - if init_val == None: - init_val = fnc(init_val) - self.val = init_val - - self.item_type = type(fnc(None)) - self.timestamp = None - # Allow us to pass in a full name to a function - if type(fnc) == str: - from boto.utils import find_class - fnc = find_class(fnc) - self.fnc = fnc - - def set(self, val): - """Set the value""" - import time - now = time.time() - expected_value = [] - new_val = {} - new_val['timestamp'] = now - if self._value != None: - new_val['last_value'] = self._value - expected_value = ['current_value', str(self._value)] - new_val['current_value'] = val - try: - self.db.put_attributes(self.id, new_val, expected_value=expected_value) - self.timestamp = new_val['timestamp'] - except SDBResponseError, e: - if e.status == 409: - raise ValueError, "Sequence out of sync" - else: - raise - - - def get(self): - """Get the value""" - val = self.db.get_attributes(self.id, consistent_read=True) - if val and val.has_key('timestamp'): - self.timestamp = val['timestamp'] - if val and val.has_key('current_value'): - self._value = self.item_type(val['current_value']) - if val.has_key("last_value") and val['last_value'] != None: - self.last_value = self.item_type(val['last_value']) - return self._value - - val = property(get, set) - - def __repr__(self): - return "%s('%s', '%s', '%s.%s', '%s')" % ( - self.__class__.__name__, - self.id, - self.domain_name, - self.fnc.__module__, self.fnc.__name__, - self.val) - - - def _connect(self): - """Connect to our domain""" - if not self._db: - if not self.domain_name: - import boto - sdb = boto.connect_sdb() - self.domain_name = boto.config.get("DB", "sequence_db", boto.config.get("DB", "db_name", "default")) - try: - self._db = sdb.get_domain(self.domain_name) - except SDBResponseError, e: - if e.status == 400: - self._db = sdb.create_domain(self.domain_name) - else: - raise - return self._db - - db = property(_connect) - - def next(self): - self.val = self.fnc(self.val, self.last_value) - return self.val - - def delete(self): - """Remove this sequence""" - self.db.delete_attributes(self.id) diff --git a/backup/src/boto/sdb/db/test_db.py b/backup/src/boto/sdb/db/test_db.py deleted file mode 100644 index 0c345ab..0000000 --- a/backup/src/boto/sdb/db/test_db.py +++ /dev/null @@ -1,225 +0,0 @@ -from boto.sdb.db.model import Model -from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty -from boto.sdb.db.property import DateTimeProperty, FloatProperty, ReferenceProperty -from boto.sdb.db.property import PasswordProperty, ListProperty, MapProperty -from datetime import datetime -import time -from boto.exception import SDBPersistenceError - -_objects = {} - -# -# This will eventually be moved to the boto.tests module and become a real unit test -# but for now it will live here. It shows examples of each of the Property types in -# use and tests the basic operations. -# -class TestBasic(Model): - - name = StringProperty() - size = IntegerProperty() - foo = BooleanProperty() - date = DateTimeProperty() - -class TestFloat(Model): - - name = StringProperty() - value = FloatProperty() - -class TestRequired(Model): - - req = StringProperty(required=True, default='foo') - -class TestReference(Model): - - ref = ReferenceProperty(reference_class=TestBasic, collection_name='refs') - -class TestSubClass(TestBasic): - - answer = IntegerProperty() - -class TestPassword(Model): - password = PasswordProperty() - -class TestList(Model): - - name = StringProperty() - nums = ListProperty(int) - -class TestMap(Model): - - name = StringProperty() - map = MapProperty() - -class TestListReference(Model): - - name = StringProperty() - basics = ListProperty(TestBasic) - -class TestAutoNow(Model): - - create_date = DateTimeProperty(auto_now_add=True) - modified_date = DateTimeProperty(auto_now=True) - -class TestUnique(Model): - name = StringProperty(unique=True) - -def test_basic(): - global _objects - t = TestBasic() - t.name = 'simple' - t.size = -42 - t.foo = True - t.date = datetime.now() - print 'saving object' - t.put() - _objects['test_basic_t'] = t - time.sleep(5) - print 'now try retrieving it' - tt = TestBasic.get_by_id(t.id) - _objects['test_basic_tt'] = tt - assert tt.id == t.id - l = TestBasic.get_by_id([t.id]) - assert len(l) == 1 - assert l[0].id == t.id - assert t.size == tt.size - assert t.foo == tt.foo - assert t.name == tt.name - #assert t.date == tt.date - return t - -def test_float(): - global _objects - t = TestFloat() - t.name = 'float object' - t.value = 98.6 - print 'saving object' - t.save() - _objects['test_float_t'] = t - time.sleep(5) - print 'now try retrieving it' - tt = TestFloat.get_by_id(t.id) - _objects['test_float_tt'] = tt - assert tt.id == t.id - assert tt.name == t.name - assert tt.value == t.value - return t - -def test_required(): - global _objects - t = TestRequired() - _objects['test_required_t'] = t - t.put() - return t - -def test_reference(t=None): - global _objects - if not t: - t = test_basic() - tt = TestReference() - tt.ref = t - tt.put() - time.sleep(10) - tt = TestReference.get_by_id(tt.id) - _objects['test_reference_tt'] = tt - assert tt.ref.id == t.id - for o in t.refs: - print o - -def test_subclass(): - global _objects - t = TestSubClass() - _objects['test_subclass_t'] = t - t.name = 'a subclass' - t.size = -489 - t.save() - -def test_password(): - global _objects - t = TestPassword() - _objects['test_password_t'] = t - t.password = "foo" - t.save() - time.sleep(5) - # Make sure it stored ok - tt = TestPassword.get_by_id(t.id) - _objects['test_password_tt'] = tt - #Testing password equality - assert tt.password == "foo" - #Testing password not stored as string - assert str(tt.password) != "foo" - -def test_list(): - global _objects - t = TestList() - _objects['test_list_t'] = t - t.name = 'a list of ints' - t.nums = [1,2,3,4,5] - t.put() - tt = TestList.get_by_id(t.id) - _objects['test_list_tt'] = tt - assert tt.name == t.name - for n in tt.nums: - assert isinstance(n, int) - -def test_list_reference(): - global _objects - t = TestBasic() - t.put() - _objects['test_list_ref_t'] = t - tt = TestListReference() - tt.name = "foo" - tt.basics = [t] - tt.put() - time.sleep(5) - _objects['test_list_ref_tt'] = tt - ttt = TestListReference.get_by_id(tt.id) - assert ttt.basics[0].id == t.id - -def test_unique(): - global _objects - t = TestUnique() - name = 'foo' + str(int(time.time())) - t.name = name - t.put() - _objects['test_unique_t'] = t - time.sleep(10) - tt = TestUnique() - _objects['test_unique_tt'] = tt - tt.name = name - try: - tt.put() - assert False - except(SDBPersistenceError): - pass - -def test_datetime(): - global _objects - t = TestAutoNow() - t.put() - _objects['test_datetime_t'] = t - time.sleep(5) - tt = TestAutoNow.get_by_id(t.id) - assert tt.create_date.timetuple() == t.create_date.timetuple() - -def test(): - print 'test_basic' - t1 = test_basic() - print 'test_required' - test_required() - print 'test_reference' - test_reference(t1) - print 'test_subclass' - test_subclass() - print 'test_password' - test_password() - print 'test_list' - test_list() - print 'test_list_reference' - test_list_reference() - print "test_datetime" - test_datetime() - print 'test_unique' - test_unique() - -if __name__ == "__main__": - test() diff --git a/backup/src/boto/sdb/domain.py b/backup/src/boto/sdb/domain.py deleted file mode 100644 index e809124..0000000 --- a/backup/src/boto/sdb/domain.py +++ /dev/null @@ -1,377 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an SDB Domain -""" -from boto.sdb.queryresultset import SelectResultSet - -class Domain: - - def __init__(self, connection=None, name=None): - self.connection = connection - self.name = name - self._metadata = None - - def __repr__(self): - return 'Domain:%s' % self.name - - def __iter__(self): - return iter(self.select("SELECT * FROM `%s`" % self.name)) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'DomainName': - self.name = value - else: - setattr(self, name, value) - - def get_metadata(self): - if not self._metadata: - self._metadata = self.connection.domain_metadata(self) - return self._metadata - - def put_attributes(self, item_name, attributes, - replace=True, expected_value=None): - """ - Store attributes for a given item. - - :type item_name: string - :param item_name: The name of the item whose attributes are being stored. - - :type attribute_names: dict or dict-like object - :param attribute_names: The name/value pairs to store as attributes - - :type expected_value: list - :param expected_value: If supplied, this is a list or tuple consisting - of a single attribute name and expected value. The list can be - of the form: - - * ['name', 'value'] - - In which case the call will first verify that the attribute - "name" of this item has a value of "value". If it does, the delete - will proceed, otherwise a ConditionalCheckFailed error will be - returned. The list can also be of the form: - - * ['name', True|False] - - which will simply check for the existence (True) or non-existence - (False) of the attribute. - - :type replace: bool - :param replace: Whether the attribute values passed in will replace - existing values or will be added as addition values. - Defaults to True. - - :rtype: bool - :return: True if successful - """ - return self.connection.put_attributes(self, item_name, attributes, - replace, expected_value) - - def batch_put_attributes(self, items, replace=True): - """ - Store attributes for multiple items. - - :type items: dict or dict-like object - :param items: A dictionary-like object. The keys of the dictionary are - the item names and the values are themselves dictionaries - of attribute names/values, exactly the same as the - attribute_names parameter of the scalar put_attributes - call. - - :type replace: bool - :param replace: Whether the attribute values passed in will replace - existing values or will be added as addition values. - Defaults to True. - - :rtype: bool - :return: True if successful - """ - return self.connection.batch_put_attributes(self, items, replace) - - def get_attributes(self, item_name, attribute_name=None, - consistent_read=False, item=None): - """ - Retrieve attributes for a given item. - - :type item_name: string - :param item_name: The name of the item whose attributes are being retrieved. - - :type attribute_names: string or list of strings - :param attribute_names: An attribute name or list of attribute names. This - parameter is optional. If not supplied, all attributes - will be retrieved for the item. - - :rtype: :class:`boto.sdb.item.Item` - :return: An Item mapping type containing the requested attribute name/values - """ - return self.connection.get_attributes(self, item_name, attribute_name, - consistent_read, item) - - def delete_attributes(self, item_name, attributes=None, - expected_values=None): - """ - Delete attributes from a given item. - - :type item_name: string - :param item_name: The name of the item whose attributes are being deleted. - - :type attributes: dict, list or :class:`boto.sdb.item.Item` - :param attributes: Either a list containing attribute names which will cause - all values associated with that attribute name to be deleted or - a dict or Item containing the attribute names and keys and list - of values to delete as the value. If no value is supplied, - all attribute name/values for the item will be deleted. - - :type expected_value: list - :param expected_value: If supplied, this is a list or tuple consisting - of a single attribute name and expected value. The list can be of - the form: - - * ['name', 'value'] - - In which case the call will first verify that the attribute "name" - of this item has a value of "value". If it does, the delete - will proceed, otherwise a ConditionalCheckFailed error will be - returned. The list can also be of the form: - - * ['name', True|False] - - which will simply check for the existence (True) or - non-existence (False) of the attribute. - - :rtype: bool - :return: True if successful - """ - return self.connection.delete_attributes(self, item_name, attributes, - expected_values) - - def batch_delete_attributes(self, items): - """ - Delete multiple items in this domain. - - :type items: dict or dict-like object - :param items: A dictionary-like object. The keys of the dictionary are - the item names and the values are either: - - * dictionaries of attribute names/values, exactly the - same as the attribute_names parameter of the scalar - put_attributes call. The attribute name/value pairs - will only be deleted if they match the name/value - pairs passed in. - * None which means that all attributes associated - with the item should be deleted. - - :rtype: bool - :return: True if successful - """ - return self.connection.batch_delete_attributes(self, items) - - def select(self, query='', next_token=None, consistent_read=False, max_items=None): - """ - Returns a set of Attributes for item names within domain_name that match the query. - The query must be expressed in using the SELECT style syntax rather than the - original SimpleDB query language. - - :type query: string - :param query: The SimpleDB query to be performed. - - :rtype: iter - :return: An iterator containing the results. This is actually a generator - function that will iterate across all search results, not just the - first page. - """ - return SelectResultSet(self, query, max_items=max_items, next_token=next_token, - consistent_read=consistent_read) - - def get_item(self, item_name, consistent_read=False): - """ - Retrieves an item from the domain, along with all of its attributes. - - :param string item_name: The name of the item to retrieve. - :rtype: :class:`boto.sdb.item.Item` or ``None`` - :keyword bool consistent_read: When set to true, ensures that the most - recent data is returned. - :return: The requested item, or ``None`` if there was no match found - """ - item = self.get_attributes(item_name, consistent_read=consistent_read) - if item: - item.domain = self - return item - else: - return None - - def new_item(self, item_name): - return self.connection.item_cls(self, item_name) - - def delete_item(self, item): - self.delete_attributes(item.name) - - def to_xml(self, f=None): - """Get this domain as an XML DOM Document - :param f: Optional File to dump directly to - :type f: File or Stream - - :return: File object where the XML has been dumped to - :rtype: file - """ - if not f: - from tempfile import TemporaryFile - f = TemporaryFile() - print >> f, '' - print >> f, '' % self.name - for item in self: - print >> f, '\t' % item.name - for k in item: - print >> f, '\t\t' % k - values = item[k] - if not isinstance(values, list): - values = [values] - for value in values: - print >> f, '\t\t\t> f, ']]>' - print >> f, '\t\t' - print >> f, '\t' - print >> f, '' - f.flush() - f.seek(0) - return f - - - def from_xml(self, doc): - """Load this domain based on an XML document""" - import xml.sax - handler = DomainDumpParser(self) - xml.sax.parse(doc, handler) - return handler - - def delete(self): - """ - Delete this domain, and all items under it - """ - return self.connection.delete(self) - - -class DomainMetaData: - - def __init__(self, domain=None): - self.domain = domain - self.item_count = None - self.item_names_size = None - self.attr_name_count = None - self.attr_names_size = None - self.attr_value_count = None - self.attr_values_size = None - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'ItemCount': - self.item_count = int(value) - elif name == 'ItemNamesSizeBytes': - self.item_names_size = int(value) - elif name == 'AttributeNameCount': - self.attr_name_count = int(value) - elif name == 'AttributeNamesSizeBytes': - self.attr_names_size = int(value) - elif name == 'AttributeValueCount': - self.attr_value_count = int(value) - elif name == 'AttributeValuesSizeBytes': - self.attr_values_size = int(value) - elif name == 'Timestamp': - self.timestamp = value - else: - setattr(self, name, value) - -import sys -from xml.sax.handler import ContentHandler -class DomainDumpParser(ContentHandler): - """ - SAX parser for a domain that has been dumped - """ - - def __init__(self, domain): - self.uploader = UploaderThread(domain) - self.item_id = None - self.attrs = {} - self.attribute = None - self.value = "" - self.domain = domain - - def startElement(self, name, attrs): - if name == "Item": - self.item_id = attrs['id'] - self.attrs = {} - elif name == "attribute": - self.attribute = attrs['id'] - elif name == "value": - self.value = "" - - def characters(self, ch): - self.value += ch - - def endElement(self, name): - if name == "value": - if self.value and self.attribute: - value = self.value.strip() - attr_name = self.attribute.strip() - if self.attrs.has_key(attr_name): - self.attrs[attr_name].append(value) - else: - self.attrs[attr_name] = [value] - elif name == "Item": - self.uploader.items[self.item_id] = self.attrs - # Every 20 items we spawn off the uploader - if len(self.uploader.items) >= 20: - self.uploader.start() - self.uploader = UploaderThread(self.domain) - elif name == "Domain": - # If we're done, spawn off our last Uploader Thread - self.uploader.start() - -from threading import Thread -class UploaderThread(Thread): - """Uploader Thread""" - - def __init__(self, domain): - self.db = domain - self.items = {} - Thread.__init__(self) - - def run(self): - try: - self.db.batch_put_attributes(self.items) - except: - print "Exception using batch put, trying regular put instead" - for item_name in self.items: - self.db.put_attributes(item_name, self.items[item_name]) - print ".", - sys.stdout.flush() diff --git a/backup/src/boto/sdb/item.py b/backup/src/boto/sdb/item.py deleted file mode 100644 index 4705b31..0000000 --- a/backup/src/boto/sdb/item.py +++ /dev/null @@ -1,183 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import base64 - -class Item(dict): - """ - A ``dict`` sub-class that serves as an object representation of a - SimpleDB item. An item in SDB is similar to a row in a relational - database. Items belong to a :py:class:`Domain `, - which is similar to a table in a relational database. - - The keys on instances of this object correspond to attributes that are - stored on the SDB item. - - .. tip:: - While it is possible to instantiate this class directly, you may want - to use the convenience methods on :py:class:`boto.sdb.domain.Domain` - for that purpose. For example, - :py:meth:`boto.sdb.domain.Domain.get_item`. - """ - def __init__(self, domain, name='', active=False): - """ - :type domain: :py:class:`boto.sdb.domain.Domain` - :param domain: The domain that this item belongs to. - - :param str name: The name of this item. This name will be used when - querying for items using methods like - :py:meth:`boto.sdb.domain.Domain.get_item` - """ - dict.__init__(self) - self.domain = domain - self.name = name - self.active = active - self.request_id = None - self.encoding = None - self.in_attribute = False - self.converter = self.domain.connection.converter - - def startElement(self, name, attrs, connection): - if name == 'Attribute': - self.in_attribute = True - self.encoding = attrs.get('encoding', None) - return None - - def decode_value(self, value): - if self.encoding == 'base64': - self.encoding = None - return base64.decodestring(value) - else: - return value - - def endElement(self, name, value, connection): - if name == 'ItemName': - self.name = self.decode_value(value) - elif name == 'Name': - if self.in_attribute: - self.last_key = self.decode_value(value) - else: - self.name = self.decode_value(value) - elif name == 'Value': - if self.has_key(self.last_key): - if not isinstance(self[self.last_key], list): - self[self.last_key] = [self[self.last_key]] - value = self.decode_value(value) - if self.converter: - value = self.converter.decode(value) - self[self.last_key].append(value) - else: - value = self.decode_value(value) - if self.converter: - value = self.converter.decode(value) - self[self.last_key] = value - elif name == 'BoxUsage': - try: - connection.box_usage += float(value) - except: - pass - elif name == 'RequestId': - self.request_id = value - elif name == 'Attribute': - self.in_attribute = False - else: - setattr(self, name, value) - - def load(self): - """ - Loads or re-loads this item's attributes from SDB. - - .. warning:: - If you have changed attribute values on an Item instance, - this method will over-write the values if they are different in - SDB. For any local attributes that don't yet exist in SDB, - they will be safe. - """ - self.domain.get_attributes(self.name, item=self) - - def save(self, replace=True): - """ - Saves this item to SDB. - - :param bool replace: If ``True``, delete any attributes on the remote - SDB item that have a ``None`` value on this object. - """ - self.domain.put_attributes(self.name, self, replace) - # Delete any attributes set to "None" - if replace: - del_attrs = [] - for name in self: - if self[name] == None: - del_attrs.append(name) - if len(del_attrs) > 0: - self.domain.delete_attributes(self.name, del_attrs) - - def add_value(self, key, value): - """ - Helps set or add to attributes on this item. If you are adding a new - attribute that has yet to be set, it will simply create an attribute - named ``key`` with your given ``value`` as its value. If you are - adding a value to an existing attribute, this method will convert the - attribute to a list (if it isn't already) and append your new value - to said list. - - For clarification, consider the following interactive session: - - .. code-block:: python - - >>> item = some_domain.get_item('some_item') - >>> item.has_key('some_attr') - False - >>> item.add_value('some_attr', 1) - >>> item['some_attr'] - 1 - >>> item.add_value('some_attr', 2) - >>> item['some_attr'] - [1, 2] - - :param str key: The attribute to add a value to. - :param object value: The value to set or append to the attribute. - """ - if key in self: - # We already have this key on the item. - if not isinstance(self[key], list): - # The key isn't already a list, take its current value and - # convert it to a list with the only member being the - # current value. - self[key] = [self[key]] - # Add the new value to the list. - self[key].append(value) - else: - # This is a new attribute, just set it. - self[key] = value - - def delete(self): - """ - Deletes this item in SDB. - - .. note:: This local Python object remains in its current state - after deletion, this only deletes the remote item in SDB. - """ - self.domain.delete_item(self) - - - - diff --git a/backup/src/boto/sdb/persist/__init__.py b/backup/src/boto/sdb/persist/__init__.py deleted file mode 100644 index 2f2b0c1..0000000 --- a/backup/src/boto/sdb/persist/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto.utils import find_class - -class Manager(object): - - DefaultDomainName = boto.config.get('Persist', 'default_domain', None) - - def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): - self.domain_name = domain_name - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - self.domain = None - self.sdb = None - self.s3 = None - if not self.domain_name: - self.domain_name = self.DefaultDomainName - if self.domain_name: - boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name) - else: - boto.log.warning('No SimpleDB domain set, persistance is disabled') - if self.domain_name: - self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id, - aws_secret_access_key=self.aws_secret_access_key, - debug=debug) - self.domain = self.sdb.lookup(self.domain_name) - if not self.domain: - self.domain = self.sdb.create_domain(self.domain_name) - - def get_s3_connection(self): - if not self.s3: - self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) - return self.s3 - -def get_manager(domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): - return Manager(domain_name, aws_access_key_id, aws_secret_access_key, debug=debug) - -def set_domain(domain_name): - Manager.DefaultDomainName = domain_name - -def get_domain(): - return Manager.DefaultDomainName - -def revive_object_from_id(id, manager): - if not manager.domain: - return None - attrs = manager.domain.get_attributes(id, ['__module__', '__type__', '__lineage__']) - try: - cls = find_class(attrs['__module__'], attrs['__type__']) - return cls(id, manager=manager) - except ImportError: - return None - -def object_lister(cls, query_lister, manager): - for item in query_lister: - if cls: - yield cls(item.name) - else: - o = revive_object_from_id(item.name, manager) - if o: - yield o - - diff --git a/backup/src/boto/sdb/persist/checker.py b/backup/src/boto/sdb/persist/checker.py deleted file mode 100644 index e2146c9..0000000 --- a/backup/src/boto/sdb/persist/checker.py +++ /dev/null @@ -1,302 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from datetime import datetime -from boto.s3.key import Key -from boto.s3.bucket import Bucket -from boto.sdb.persist import revive_object_from_id -from boto.exception import SDBPersistenceError -from boto.utils import Password - -ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - -class ValueChecker: - - def check(self, value): - """ - Checks a value to see if it is of the right type. - - Should raise a TypeError exception if an in appropriate value is passed in. - """ - raise TypeError - - def from_string(self, str_value, obj): - """ - Takes a string as input and returns the type-specific value represented by that string. - - Should raise a ValueError if the value cannot be converted to the appropriate type. - """ - raise ValueError - - def to_string(self, value): - """ - Convert a value to it's string representation. - - Should raise a ValueError if the value cannot be converted to a string representation. - """ - raise ValueError - -class StringChecker(ValueChecker): - - def __init__(self, **params): - if params.has_key('maxlength'): - self.maxlength = params['maxlength'] - else: - self.maxlength = 1024 - if params.has_key('default'): - self.check(params['default']) - self.default = params['default'] - else: - self.default = '' - - def check(self, value): - if isinstance(value, str) or isinstance(value, unicode): - if len(value) > self.maxlength: - raise ValueError, 'Length of value greater than maxlength' - else: - raise TypeError, 'Expecting String, got %s' % type(value) - - def from_string(self, str_value, obj): - return str_value - - def to_string(self, value): - self.check(value) - return value - -class PasswordChecker(StringChecker): - def check(self, value): - if isinstance(value, str) or isinstance(value, unicode) or isinstance(value, Password): - if len(value) > self.maxlength: - raise ValueError, 'Length of value greater than maxlength' - else: - raise TypeError, 'Expecting String, got %s' % type(value) - -class IntegerChecker(ValueChecker): - - __sizes__ = { 'small' : (65535, 32767, -32768, 5), - 'medium' : (4294967295, 2147483647, -2147483648, 10), - 'large' : (18446744073709551615, 9223372036854775807, -9223372036854775808, 20)} - - def __init__(self, **params): - self.size = params.get('size', 'medium') - if self.size not in self.__sizes__.keys(): - raise ValueError, 'size must be one of %s' % self.__sizes__.keys() - self.signed = params.get('signed', True) - self.default = params.get('default', 0) - self.format_string = '%%0%dd' % self.__sizes__[self.size][-1] - - def check(self, value): - if not isinstance(value, int) and not isinstance(value, long): - raise TypeError, 'Expecting int or long, got %s' % type(value) - if self.signed: - min = self.__sizes__[self.size][2] - max = self.__sizes__[self.size][1] - else: - min = 0 - max = self.__sizes__[self.size][0] - if value > max: - raise ValueError, 'Maximum value is %d' % max - if value < min: - raise ValueError, 'Minimum value is %d' % min - - def from_string(self, str_value, obj): - val = int(str_value) - if self.signed: - val = val + self.__sizes__[self.size][2] - return val - - def to_string(self, value): - self.check(value) - if self.signed: - value += -self.__sizes__[self.size][2] - return self.format_string % value - -class BooleanChecker(ValueChecker): - - def __init__(self, **params): - if params.has_key('default'): - self.default = params['default'] - else: - self.default = False - - def check(self, value): - if not isinstance(value, bool): - raise TypeError, 'Expecting bool, got %s' % type(value) - - def from_string(self, str_value, obj): - if str_value.lower() == 'true': - return True - else: - return False - - def to_string(self, value): - self.check(value) - if value == True: - return 'true' - else: - return 'false' - -class DateTimeChecker(ValueChecker): - - def __init__(self, **params): - if params.has_key('maxlength'): - self.maxlength = params['maxlength'] - else: - self.maxlength = 1024 - if params.has_key('default'): - self.default = params['default'] - else: - self.default = datetime.now() - - def check(self, value): - if not isinstance(value, datetime): - raise TypeError, 'Expecting datetime, got %s' % type(value) - - def from_string(self, str_value, obj): - try: - return datetime.strptime(str_value, ISO8601) - except: - raise ValueError, 'Unable to convert %s to DateTime' % str_value - - def to_string(self, value): - self.check(value) - return value.strftime(ISO8601) - -class ObjectChecker(ValueChecker): - - def __init__(self, **params): - self.default = None - self.ref_class = params.get('ref_class', None) - if self.ref_class == None: - raise SDBPersistenceError('ref_class parameter is required') - - def check(self, value): - if value == None: - return - if isinstance(value, str) or isinstance(value, unicode): - # ugly little hack - sometimes I want to just stick a UUID string - # in here rather than instantiate an object. - # This does a bit of hand waving to "type check" the string - t = value.split('-') - if len(t) != 5: - raise ValueError - else: - try: - obj_lineage = value.get_lineage() - cls_lineage = self.ref_class.get_lineage() - if obj_lineage.startswith(cls_lineage): - return - raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) - except: - raise ValueError, '%s is not an SDBObject' % value - - def from_string(self, str_value, obj): - if not str_value: - return None - try: - return revive_object_from_id(str_value, obj._manager) - except: - raise ValueError, 'Unable to convert %s to Object' % str_value - - def to_string(self, value): - self.check(value) - if isinstance(value, str) or isinstance(value, unicode): - return value - if value == None: - return '' - else: - return value.id - -class S3KeyChecker(ValueChecker): - - def __init__(self, **params): - self.default = None - - def check(self, value): - if value == None: - return - if isinstance(value, str) or isinstance(value, unicode): - try: - bucket_name, key_name = value.split('/', 1) - except: - raise ValueError - elif not isinstance(value, Key): - raise TypeError, 'Expecting Key, got %s' % type(value) - - def from_string(self, str_value, obj): - if not str_value: - return None - if str_value == 'None': - return None - try: - bucket_name, key_name = str_value.split('/', 1) - if obj: - s3 = obj._manager.get_s3_connection() - bucket = s3.get_bucket(bucket_name) - key = bucket.get_key(key_name) - if not key: - key = bucket.new_key(key_name) - return key - except: - raise ValueError, 'Unable to convert %s to S3Key' % str_value - - def to_string(self, value): - self.check(value) - if isinstance(value, str) or isinstance(value, unicode): - return value - if value == None: - return '' - else: - return '%s/%s' % (value.bucket.name, value.name) - -class S3BucketChecker(ValueChecker): - - def __init__(self, **params): - self.default = None - - def check(self, value): - if value == None: - return - if isinstance(value, str) or isinstance(value, unicode): - return - elif not isinstance(value, Bucket): - raise TypeError, 'Expecting Bucket, got %s' % type(value) - - def from_string(self, str_value, obj): - if not str_value: - return None - if str_value == 'None': - return None - try: - if obj: - s3 = obj._manager.get_s3_connection() - bucket = s3.get_bucket(str_value) - return bucket - except: - raise ValueError, 'Unable to convert %s to S3Bucket' % str_value - - def to_string(self, value): - self.check(value) - if value == None: - return '' - else: - return '%s' % value.name - diff --git a/backup/src/boto/sdb/persist/object.py b/backup/src/boto/sdb/persist/object.py deleted file mode 100644 index 993df1e..0000000 --- a/backup/src/boto/sdb/persist/object.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.exception import SDBPersistenceError -from boto.sdb.persist import get_manager, object_lister -from boto.sdb.persist.property import Property, ScalarProperty -import uuid - -class SDBBase(type): - "Metaclass for all SDBObjects" - def __init__(cls, name, bases, dict): - super(SDBBase, cls).__init__(name, bases, dict) - # Make sure this is a subclass of SDBObject - mainly copied from django ModelBase (thanks!) - try: - if filter(lambda b: issubclass(b, SDBObject), bases): - # look for all of the Properties and set their names - for key in dict.keys(): - if isinstance(dict[key], Property): - property = dict[key] - property.set_name(key) - prop_names = [] - props = cls.properties() - for prop in props: - prop_names.append(prop.name) - setattr(cls, '_prop_names', prop_names) - except NameError: - # 'SDBObject' isn't defined yet, meaning we're looking at our own - # SDBObject class, defined below. - pass - -class SDBObject(object): - __metaclass__ = SDBBase - - _manager = get_manager() - - @classmethod - def get_lineage(cls): - l = [c.__name__ for c in cls.mro()] - l.reverse() - return '.'.join(l) - - @classmethod - def get(cls, id=None, **params): - if params.has_key('manager'): - manager = params['manager'] - else: - manager = cls._manager - if manager.domain and id: - a = cls._manager.domain.get_attributes(id, '__type__') - if a.has_key('__type__'): - return cls(id, manager) - else: - raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) - else: - rs = cls.find(**params) - try: - obj = rs.next() - except StopIteration: - raise SDBPersistenceError('%s object matching query does not exist' % cls.__name__) - try: - rs.next() - except StopIteration: - return obj - raise SDBPersistenceError('Query matched more than 1 item') - - @classmethod - def find(cls, **params): - if params.has_key('manager'): - manager = params['manager'] - del params['manager'] - else: - manager = cls._manager - keys = params.keys() - if len(keys) > 4: - raise SDBPersistenceError('Too many fields, max is 4') - parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())] - properties = cls.properties() - for key in keys: - found = False - for property in properties: - if property.name == key: - found = True - if isinstance(property, ScalarProperty): - checker = property.checker - parts.append("['%s' = '%s']" % (key, checker.to_string(params[key]))) - else: - raise SDBPersistenceError('%s is not a searchable field' % key) - if not found: - raise SDBPersistenceError('%s is not a valid field' % key) - query = ' intersection '.join(parts) - if manager.domain: - rs = manager.domain.query(query) - else: - rs = [] - return object_lister(None, rs, manager) - - @classmethod - def list(cls, max_items=None, manager=None): - if not manager: - manager = cls._manager - if manager.domain: - rs = manager.domain.query("['__type__' = '%s']" % cls.__name__, max_items=max_items) - else: - rs = [] - return object_lister(cls, rs, manager) - - @classmethod - def properties(cls): - properties = [] - while cls: - for key in cls.__dict__.keys(): - if isinstance(cls.__dict__[key], Property): - properties.append(cls.__dict__[key]) - if len(cls.__bases__) > 0: - cls = cls.__bases__[0] - else: - cls = None - return properties - - # for backwards compatibility - find_properties = properties - - def __init__(self, id=None, manager=None): - if manager: - self._manager = manager - self.id = id - if self.id: - self._auto_update = True - if self._manager.domain: - attrs = self._manager.domain.get_attributes(self.id, '__type__') - if len(attrs.keys()) == 0: - raise SDBPersistenceError('Object %s: not found' % self.id) - else: - self.id = str(uuid.uuid4()) - self._auto_update = False - - def __setattr__(self, name, value): - if name in self._prop_names: - object.__setattr__(self, name, value) - elif name.startswith('_'): - object.__setattr__(self, name, value) - elif name == 'id': - object.__setattr__(self, name, value) - else: - self._persist_attribute(name, value) - object.__setattr__(self, name, value) - - def __getattr__(self, name): - if not name.startswith('_'): - a = self._manager.domain.get_attributes(self.id, name) - if a.has_key(name): - object.__setattr__(self, name, a[name]) - return a[name] - raise AttributeError - - def __repr__(self): - return '%s<%s>' % (self.__class__.__name__, self.id) - - def _persist_attribute(self, name, value): - if self.id: - self._manager.domain.put_attributes(self.id, {name : value}, replace=True) - - def _get_sdb_item(self): - return self._manager.domain.get_item(self.id) - - def save(self): - attrs = {'__type__' : self.__class__.__name__, - '__module__' : self.__class__.__module__, - '__lineage__' : self.get_lineage()} - for property in self.properties(): - attrs[property.name] = property.to_string(self) - if self._manager.domain: - self._manager.domain.put_attributes(self.id, attrs, replace=True) - self._auto_update = True - - def delete(self): - if self._manager.domain: - self._manager.domain.delete_attributes(self.id) - - def get_related_objects(self, ref_name, ref_cls=None): - if self._manager.domain: - query = "['%s' = '%s']" % (ref_name, self.id) - if ref_cls: - query += " intersection ['__type__'='%s']" % ref_cls.__name__ - rs = self._manager.domain.query(query) - else: - rs = [] - return object_lister(ref_cls, rs, self._manager) - diff --git a/backup/src/boto/sdb/persist/property.py b/backup/src/boto/sdb/persist/property.py deleted file mode 100644 index 4776d35..0000000 --- a/backup/src/boto/sdb/persist/property.py +++ /dev/null @@ -1,371 +0,0 @@ -# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.exception import SDBPersistenceError -from boto.sdb.persist.checker import StringChecker, PasswordChecker, IntegerChecker, BooleanChecker -from boto.sdb.persist.checker import DateTimeChecker, ObjectChecker, S3KeyChecker, S3BucketChecker -from boto.utils import Password - -class Property(object): - - def __init__(self, checker_class, **params): - self.name = '' - self.checker = checker_class(**params) - self.slot_name = '__' - - def set_name(self, name): - self.name = name - self.slot_name = '__' + self.name - -class ScalarProperty(Property): - - def save(self, obj): - domain = obj._manager.domain - domain.put_attributes(obj.id, {self.name : self.to_string(obj)}, replace=True) - - def to_string(self, obj): - return self.checker.to_string(getattr(obj, self.name)) - - def load(self, obj): - domain = obj._manager.domain - a = domain.get_attributes(obj.id, self.name) - # try to get the attribute value from SDB - if self.name in a: - value = self.checker.from_string(a[self.name], obj) - setattr(obj, self.slot_name, value) - # if it's not there, set the value to the default value - else: - self.__set__(obj, self.checker.default) - - def __get__(self, obj, objtype): - if obj: - try: - value = getattr(obj, self.slot_name) - except AttributeError: - if obj._auto_update: - self.load(obj) - value = getattr(obj, self.slot_name) - else: - value = self.checker.default - setattr(obj, self.slot_name, self.checker.default) - return value - - def __set__(self, obj, value): - self.checker.check(value) - try: - old_value = getattr(obj, self.slot_name) - except: - old_value = self.checker.default - setattr(obj, self.slot_name, value) - if obj._auto_update: - try: - self.save(obj) - except: - setattr(obj, self.slot_name, old_value) - raise - -class StringProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, StringChecker, **params) - -class PasswordProperty(ScalarProperty): - """ - Hashed password - """ - - def __init__(self, **params): - ScalarProperty.__init__(self, PasswordChecker, **params) - - def __set__(self, obj, value): - p = Password() - p.set(value) - ScalarProperty.__set__(self, obj, p) - - def __get__(self, obj, objtype): - return Password(ScalarProperty.__get__(self, obj, objtype)) - -class SmallPositiveIntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'small' - params['signed'] = False - ScalarProperty.__init__(self, IntegerChecker, **params) - -class SmallIntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'small' - params['signed'] = True - ScalarProperty.__init__(self, IntegerChecker, **params) - -class PositiveIntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'medium' - params['signed'] = False - ScalarProperty.__init__(self, IntegerChecker, **params) - -class IntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'medium' - params['signed'] = True - ScalarProperty.__init__(self, IntegerChecker, **params) - -class LargePositiveIntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'large' - params['signed'] = False - ScalarProperty.__init__(self, IntegerChecker, **params) - -class LargeIntegerProperty(ScalarProperty): - - def __init__(self, **params): - params['size'] = 'large' - params['signed'] = True - ScalarProperty.__init__(self, IntegerChecker, **params) - -class BooleanProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, BooleanChecker, **params) - -class DateTimeProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, DateTimeChecker, **params) - -class ObjectProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, ObjectChecker, **params) - -class S3KeyProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, S3KeyChecker, **params) - - def __set__(self, obj, value): - self.checker.check(value) - try: - old_value = getattr(obj, self.slot_name) - except: - old_value = self.checker.default - if isinstance(value, str): - value = self.checker.from_string(value, obj) - setattr(obj, self.slot_name, value) - if obj._auto_update: - try: - self.save(obj) - except: - setattr(obj, self.slot_name, old_value) - raise - -class S3BucketProperty(ScalarProperty): - - def __init__(self, **params): - ScalarProperty.__init__(self, S3BucketChecker, **params) - - def __set__(self, obj, value): - self.checker.check(value) - try: - old_value = getattr(obj, self.slot_name) - except: - old_value = self.checker.default - if isinstance(value, str): - value = self.checker.from_string(value, obj) - setattr(obj, self.slot_name, value) - if obj._auto_update: - try: - self.save(obj) - except: - setattr(obj, self.slot_name, old_value) - raise - -class MultiValueProperty(Property): - - def __init__(self, checker_class, **params): - Property.__init__(self, checker_class, **params) - - def __get__(self, obj, objtype): - if obj: - try: - value = getattr(obj, self.slot_name) - except AttributeError: - if obj._auto_update: - self.load(obj) - value = getattr(obj, self.slot_name) - else: - value = MultiValue(self, obj, []) - setattr(obj, self.slot_name, value) - return value - - def load(self, obj): - if obj != None: - _list = [] - domain = obj._manager.domain - a = domain.get_attributes(obj.id, self.name) - if self.name in a: - lst = a[self.name] - if not isinstance(lst, list): - lst = [lst] - for value in lst: - value = self.checker.from_string(value, obj) - _list.append(value) - setattr(obj, self.slot_name, MultiValue(self, obj, _list)) - - def __set__(self, obj, value): - if not isinstance(value, list): - raise SDBPersistenceError('Value must be a list') - setattr(obj, self.slot_name, MultiValue(self, obj, value)) - str_list = self.to_string(obj) - domain = obj._manager.domain - if obj._auto_update: - if len(str_list) == 1: - domain.put_attributes(obj.id, {self.name : str_list[0]}, replace=True) - else: - try: - self.__delete__(obj) - except: - pass - domain.put_attributes(obj.id, {self.name : str_list}, replace=True) - setattr(obj, self.slot_name, MultiValue(self, obj, value)) - - def __delete__(self, obj): - if obj._auto_update: - domain = obj._manager.domain - domain.delete_attributes(obj.id, [self.name]) - setattr(obj, self.slot_name, MultiValue(self, obj, [])) - - def to_string(self, obj): - str_list = [] - for value in self.__get__(obj, type(obj)): - str_list.append(self.checker.to_string(value)) - return str_list - -class StringListProperty(MultiValueProperty): - - def __init__(self, **params): - MultiValueProperty.__init__(self, StringChecker, **params) - -class SmallIntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'small' - params['signed'] = True - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class SmallPositiveIntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'small' - params['signed'] = False - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class IntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'medium' - params['signed'] = True - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class PositiveIntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'medium' - params['signed'] = False - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class LargeIntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'large' - params['signed'] = True - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class LargePositiveIntegerListProperty(MultiValueProperty): - - def __init__(self, **params): - params['size'] = 'large' - params['signed'] = False - MultiValueProperty.__init__(self, IntegerChecker, **params) - -class BooleanListProperty(MultiValueProperty): - - def __init__(self, **params): - MultiValueProperty.__init__(self, BooleanChecker, **params) - -class ObjectListProperty(MultiValueProperty): - - def __init__(self, **params): - MultiValueProperty.__init__(self, ObjectChecker, **params) - -class HasManyProperty(Property): - - def set_name(self, name): - self.name = name - self.slot_name = '__' + self.name - - def __get__(self, obj, objtype): - return self - - -class MultiValue: - """ - Special Multi Value for boto persistence layer to allow us to do - obj.list.append(foo) - """ - def __init__(self, property, obj, _list): - self.checker = property.checker - self.name = property.name - self.object = obj - self._list = _list - - def __repr__(self): - return repr(self._list) - - def __getitem__(self, key): - return self._list.__getitem__(key) - - def __delitem__(self, key): - item = self[key] - self._list.__delitem__(key) - domain = self.object._manager.domain - domain.delete_attributes(self.object.id, {self.name: [self.checker.to_string(item)]}) - - def __len__(self): - return len(self._list) - - def append(self, value): - self.checker.check(value) - self._list.append(value) - domain = self.object._manager.domain - domain.put_attributes(self.object.id, {self.name: self.checker.to_string(value)}, replace=False) - - def index(self, value): - for x in self._list: - if x.id == value.id: - return self._list.index(x) - - def remove(self, value): - del(self[self.index(value)]) diff --git a/backup/src/boto/sdb/persist/test_persist.py b/backup/src/boto/sdb/persist/test_persist.py deleted file mode 100644 index 080935d..0000000 --- a/backup/src/boto/sdb/persist/test_persist.py +++ /dev/null @@ -1,141 +0,0 @@ -from boto.sdb.persist.object import SDBObject -from boto.sdb.persist.property import StringProperty, PositiveIntegerProperty, IntegerProperty -from boto.sdb.persist.property import BooleanProperty, DateTimeProperty, S3KeyProperty -from boto.sdb.persist.property import ObjectProperty, StringListProperty -from boto.sdb.persist.property import PositiveIntegerListProperty, BooleanListProperty, ObjectListProperty -from boto.sdb.persist import Manager -from datetime import datetime -import time - -# -# This will eventually be moved to the boto.tests module and become a real unit test -# but for now it will live here. It shows examples of each of the Property types in -# use and tests the basic operations. -# -class TestScalar(SDBObject): - - name = StringProperty() - description = StringProperty() - size = PositiveIntegerProperty() - offset = IntegerProperty() - foo = BooleanProperty() - date = DateTimeProperty() - file = S3KeyProperty() - -class TestRef(SDBObject): - - name = StringProperty() - ref = ObjectProperty(ref_class=TestScalar) - -class TestSubClass1(TestRef): - - answer = PositiveIntegerProperty() - -class TestSubClass2(TestScalar): - - flag = BooleanProperty() - -class TestList(SDBObject): - - names = StringListProperty() - numbers = PositiveIntegerListProperty() - bools = BooleanListProperty() - objects = ObjectListProperty(ref_class=TestScalar) - -def test1(): - s = TestScalar() - s.name = 'foo' - s.description = 'This is foo' - s.size = 42 - s.offset = -100 - s.foo = True - s.date = datetime.now() - s.save() - return s - -def test2(ref_name): - s = TestRef() - s.name = 'testref' - rs = TestScalar.find(name=ref_name) - s.ref = rs.next() - s.save() - return s - -def test3(): - s = TestScalar() - s.name = 'bar' - s.description = 'This is bar' - s.size = 24 - s.foo = False - s.date = datetime.now() - s.save() - return s - -def test4(ref1, ref2): - s = TestList() - s.names.append(ref1.name) - s.names.append(ref2.name) - s.numbers.append(ref1.size) - s.numbers.append(ref2.size) - s.bools.append(ref1.foo) - s.bools.append(ref2.foo) - s.objects.append(ref1) - s.objects.append(ref2) - s.save() - return s - -def test5(ref): - s = TestSubClass1() - s.answer = 42 - s.ref = ref - s.save() - # test out free form attribute - s.fiddlefaddle = 'this is fiddlefaddle' - s._fiddlefaddle = 'this is not fiddlefaddle' - return s - -def test6(): - s = TestSubClass2() - s.name = 'fie' - s.description = 'This is fie' - s.size = 4200 - s.offset = -820 - s.foo = False - s.date = datetime.now() - s.flag = True - s.save() - return s - -def test(domain_name): - print 'Initialize the Persistance system' - Manager.DefaultDomainName = domain_name - print 'Call test1' - s1 = test1() - # now create a new instance and read the saved data from SDB - print 'Now sleep to wait for things to converge' - time.sleep(5) - print 'Now lookup the object and compare the fields' - s2 = TestScalar(s1.id) - assert s1.name == s2.name - assert s1.description == s2.description - assert s1.size == s2.size - assert s1.offset == s2.offset - assert s1.foo == s2.foo - #assert s1.date == s2.date - print 'Call test2' - s2 = test2(s1.name) - print 'Call test3' - s3 = test3() - print 'Call test4' - s4 = test4(s1, s3) - print 'Call test5' - s6 = test6() - s5 = test5(s6) - domain = s5._manager.domain - item1 = domain.get_item(s1.id) - item2 = domain.get_item(s2.id) - item3 = domain.get_item(s3.id) - item4 = domain.get_item(s4.id) - item5 = domain.get_item(s5.id) - item6 = domain.get_item(s6.id) - return [(s1, item1), (s2, item2), (s3, item3), (s4, item4), (s5, item5), (s6, item6)] diff --git a/backup/src/boto/sdb/queryresultset.py b/backup/src/boto/sdb/queryresultset.py deleted file mode 100644 index 10bafd1..0000000 --- a/backup/src/boto/sdb/queryresultset.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -def query_lister(domain, query='', max_items=None, attr_names=None): - more_results = True - num_results = 0 - next_token = None - while more_results: - rs = domain.connection.query_with_attributes(domain, query, attr_names, - next_token=next_token) - for item in rs: - if max_items: - if num_results == max_items: - raise StopIteration - yield item - num_results += 1 - next_token = rs.next_token - more_results = next_token != None - -class QueryResultSet: - - def __init__(self, domain=None, query='', max_items=None, attr_names=None): - self.max_items = max_items - self.domain = domain - self.query = query - self.attr_names = attr_names - - def __iter__(self): - return query_lister(self.domain, self.query, self.max_items, self.attr_names) - -def select_lister(domain, query='', max_items=None): - more_results = True - num_results = 0 - next_token = None - while more_results: - rs = domain.connection.select(domain, query, next_token=next_token) - for item in rs: - if max_items: - if num_results == max_items: - raise StopIteration - yield item - num_results += 1 - next_token = rs.next_token - more_results = next_token != None - -class SelectResultSet(object): - - def __init__(self, domain=None, query='', max_items=None, - next_token=None, consistent_read=False): - self.domain = domain - self.query = query - self.consistent_read = consistent_read - self.max_items = max_items - self.next_token = next_token - - def __iter__(self): - more_results = True - num_results = 0 - while more_results: - rs = self.domain.connection.select(self.domain, self.query, - next_token=self.next_token, - consistent_read=self.consistent_read) - for item in rs: - if self.max_items and num_results >= self.max_items: - raise StopIteration - yield item - num_results += 1 - self.next_token = rs.next_token - if self.max_items and num_results >= self.max_items: - raise StopIteration - more_results = self.next_token != None - - def next(self): - return self.__iter__().next() diff --git a/backup/src/boto/sdb/regioninfo.py b/backup/src/boto/sdb/regioninfo.py deleted file mode 100644 index 5c32864..0000000 --- a/backup/src/boto/sdb/regioninfo.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from boto.regioninfo import RegionInfo - -class SDBRegionInfo(RegionInfo): - - def __init__(self, connection=None, name=None, endpoint=None): - from boto.sdb.connection import SDBConnection - RegionInfo.__init__(self, connection, name, endpoint, - SDBConnection) diff --git a/backup/src/boto/services/__init__.py b/backup/src/boto/services/__init__.py deleted file mode 100644 index 449bd16..0000000 --- a/backup/src/boto/services/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - - diff --git a/backup/src/boto/services/bs.py b/backup/src/boto/services/bs.py deleted file mode 100644 index 3d70031..0000000 --- a/backup/src/boto/services/bs.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -from optparse import OptionParser -from boto.services.servicedef import ServiceDef -from boto.services.submit import Submitter -from boto.services.result import ResultProcessor -import boto -import sys, os, StringIO - -class BS(object): - - Usage = "usage: %prog [options] config_file command" - - Commands = {'reset' : 'Clear input queue and output bucket', - 'submit' : 'Submit local files to the service', - 'start' : 'Start the service', - 'status' : 'Report on the status of the service buckets and queues', - 'retrieve' : 'Retrieve output generated by a batch', - 'batches' : 'List all batches stored in current output_domain'} - - def __init__(self): - self.service_name = None - self.parser = OptionParser(usage=self.Usage) - self.parser.add_option("--help-commands", action="store_true", dest="help_commands", - help="provides help on the available commands") - self.parser.add_option("-a", "--access-key", action="store", type="string", - help="your AWS Access Key") - self.parser.add_option("-s", "--secret-key", action="store", type="string", - help="your AWS Secret Access Key") - self.parser.add_option("-p", "--path", action="store", type="string", dest="path", - help="the path to local directory for submit and retrieve") - self.parser.add_option("-k", "--keypair", action="store", type="string", dest="keypair", - help="the SSH keypair used with launched instance(s)") - self.parser.add_option("-l", "--leave", action="store_true", dest="leave", - help="leave the files (don't retrieve) files during retrieve command") - self.parser.set_defaults(leave=False) - self.parser.add_option("-n", "--num-instances", action="store", type="string", dest="num_instances", - help="the number of launched instance(s)") - self.parser.set_defaults(num_instances=1) - self.parser.add_option("-i", "--ignore-dirs", action="append", type="string", dest="ignore", - help="directories that should be ignored by submit command") - self.parser.add_option("-b", "--batch-id", action="store", type="string", dest="batch", - help="batch identifier required by the retrieve command") - - def print_command_help(self): - print '\nCommands:' - for key in self.Commands.keys(): - print ' %s\t\t%s' % (key, self.Commands[key]) - - def do_reset(self): - iq = self.sd.get_obj('input_queue') - if iq: - print 'clearing out input queue' - i = 0 - m = iq.read() - while m: - i += 1 - iq.delete_message(m) - m = iq.read() - print 'deleted %d messages' % i - ob = self.sd.get_obj('output_bucket') - ib = self.sd.get_obj('input_bucket') - if ob: - if ib and ob.name == ib.name: - return - print 'delete generated files in output bucket' - i = 0 - for k in ob: - i += 1 - k.delete() - print 'deleted %d keys' % i - - def do_submit(self): - if not self.options.path: - self.parser.error('No path provided') - if not os.path.exists(self.options.path): - self.parser.error('Invalid path (%s)' % self.options.path) - s = Submitter(self.sd) - t = s.submit_path(self.options.path, None, self.options.ignore, None, - None, True, self.options.path) - print 'A total of %d files were submitted' % t[1] - print 'Batch Identifier: %s' % t[0] - - def do_start(self): - ami_id = self.sd.get('ami_id') - instance_type = self.sd.get('instance_type', 'm1.small') - security_group = self.sd.get('security_group', 'default') - if not ami_id: - self.parser.error('ami_id option is required when starting the service') - ec2 = boto.connect_ec2() - if not self.sd.has_section('Credentials'): - self.sd.add_section('Credentials') - self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id) - self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key) - s = StringIO.StringIO() - self.sd.write(s) - rs = ec2.get_all_images([ami_id]) - img = rs[0] - r = img.run(user_data=s.getvalue(), key_name=self.options.keypair, - max_count=self.options.num_instances, - instance_type=instance_type, - security_groups=[security_group]) - print 'Starting AMI: %s' % ami_id - print 'Reservation %s contains the following instances:' % r.id - for i in r.instances: - print '\t%s' % i.id - - def do_status(self): - iq = self.sd.get_obj('input_queue') - if iq: - print 'The input_queue (%s) contains approximately %s messages' % (iq.id, iq.count()) - ob = self.sd.get_obj('output_bucket') - ib = self.sd.get_obj('input_bucket') - if ob: - if ib and ob.name == ib.name: - return - total = 0 - for k in ob: - total += 1 - print 'The output_bucket (%s) contains %d keys' % (ob.name, total) - - def do_retrieve(self): - if not self.options.path: - self.parser.error('No path provided') - if not os.path.exists(self.options.path): - self.parser.error('Invalid path (%s)' % self.options.path) - if not self.options.batch: - self.parser.error('batch identifier is required for retrieve command') - s = ResultProcessor(self.options.batch, self.sd) - s.get_results(self.options.path, get_file=(not self.options.leave)) - - def do_batches(self): - d = self.sd.get_obj('output_domain') - if d: - print 'Available Batches:' - rs = d.query("['type'='Batch']") - for item in rs: - print ' %s' % item.name - else: - self.parser.error('No output_domain specified for service') - - def main(self): - self.options, self.args = self.parser.parse_args() - if self.options.help_commands: - self.print_command_help() - sys.exit(0) - if len(self.args) != 2: - self.parser.error("config_file and command are required") - self.config_file = self.args[0] - self.sd = ServiceDef(self.config_file) - self.command = self.args[1] - if hasattr(self, 'do_%s' % self.command): - method = getattr(self, 'do_%s' % self.command) - method() - else: - self.parser.error('command (%s) not recognized' % self.command) - -if __name__ == "__main__": - bs = BS() - bs.main() diff --git a/backup/src/boto/services/message.py b/backup/src/boto/services/message.py deleted file mode 100644 index 79f6d19..0000000 --- a/backup/src/boto/services/message.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.sqs.message import MHMessage -from boto.utils import get_ts -from socket import gethostname -import os, mimetypes, time - -class ServiceMessage(MHMessage): - - def for_key(self, key, params=None, bucket_name=None): - if params: - self.update(params) - if key.path: - t = os.path.split(key.path) - self['OriginalLocation'] = t[0] - self['OriginalFileName'] = t[1] - mime_type = mimetypes.guess_type(t[1])[0] - if mime_type == None: - mime_type = 'application/octet-stream' - self['Content-Type'] = mime_type - s = os.stat(key.path) - t = time.gmtime(s[7]) - self['FileAccessedDate'] = get_ts(t) - t = time.gmtime(s[8]) - self['FileModifiedDate'] = get_ts(t) - t = time.gmtime(s[9]) - self['FileCreateDate'] = get_ts(t) - else: - self['OriginalFileName'] = key.name - self['OriginalLocation'] = key.bucket.name - self['ContentType'] = key.content_type - self['Host'] = gethostname() - if bucket_name: - self['Bucket'] = bucket_name - else: - self['Bucket'] = key.bucket.name - self['InputKey'] = key.name - self['Size'] = key.size - diff --git a/backup/src/boto/services/result.py b/backup/src/boto/services/result.py deleted file mode 100644 index 32a6d6a..0000000 --- a/backup/src/boto/services/result.py +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import os -from datetime import datetime, timedelta -from boto.utils import parse_ts -import boto - -class ResultProcessor: - - LogFileName = 'log.csv' - - def __init__(self, batch_name, sd, mimetype_files=None): - self.sd = sd - self.batch = batch_name - self.log_fp = None - self.num_files = 0 - self.total_time = 0 - self.min_time = timedelta.max - self.max_time = timedelta.min - self.earliest_time = datetime.max - self.latest_time = datetime.min - self.queue = self.sd.get_obj('output_queue') - self.domain = self.sd.get_obj('output_domain') - - def calculate_stats(self, msg): - start_time = parse_ts(msg['Service-Read']) - end_time = parse_ts(msg['Service-Write']) - elapsed_time = end_time - start_time - if elapsed_time > self.max_time: - self.max_time = elapsed_time - if elapsed_time < self.min_time: - self.min_time = elapsed_time - self.total_time += elapsed_time.seconds - if start_time < self.earliest_time: - self.earliest_time = start_time - if end_time > self.latest_time: - self.latest_time = end_time - - def log_message(self, msg, path): - keys = msg.keys() - keys.sort() - if not self.log_fp: - self.log_fp = open(os.path.join(path, self.LogFileName), 'a') - line = ','.join(keys) - self.log_fp.write(line+'\n') - values = [] - for key in keys: - value = msg[key] - if value.find(',') > 0: - value = '"%s"' % value - values.append(value) - line = ','.join(values) - self.log_fp.write(line+'\n') - - def process_record(self, record, path, get_file=True): - self.log_message(record, path) - self.calculate_stats(record) - outputs = record['OutputKey'].split(',') - if record.has_key('OutputBucket'): - bucket = boto.lookup('s3', record['OutputBucket']) - else: - bucket = boto.lookup('s3', record['Bucket']) - for output in outputs: - if get_file: - key_name = output.split(';')[0] - key = bucket.lookup(key_name) - file_name = os.path.join(path, key_name) - print 'retrieving file: %s to %s' % (key_name, file_name) - key.get_contents_to_filename(file_name) - self.num_files += 1 - - def get_results_from_queue(self, path, get_file=True, delete_msg=True): - m = self.queue.read() - while m: - if m.has_key('Batch') and m['Batch'] == self.batch: - self.process_record(m, path, get_file) - if delete_msg: - self.queue.delete_message(m) - m = self.queue.read() - - def get_results_from_domain(self, path, get_file=True): - rs = self.domain.query("['Batch'='%s']" % self.batch) - for item in rs: - self.process_record(item, path, get_file) - - def get_results_from_bucket(self, path): - bucket = self.sd.get_obj('output_bucket') - if bucket: - print 'No output queue or domain, just retrieving files from output_bucket' - for key in bucket: - file_name = os.path.join(path, key) - print 'retrieving file: %s to %s' % (key, file_name) - key.get_contents_to_filename(file_name) - self.num_files + 1 - - def get_results(self, path, get_file=True, delete_msg=True): - if not os.path.isdir(path): - os.mkdir(path) - if self.queue: - self.get_results_from_queue(path, get_file) - elif self.domain: - self.get_results_from_domain(path, get_file) - else: - self.get_results_from_bucket(path) - if self.log_fp: - self.log_fp.close() - print '%d results successfully retrieved.' % self.num_files - if self.num_files > 0: - self.avg_time = float(self.total_time)/self.num_files - print 'Minimum Processing Time: %d' % self.min_time.seconds - print 'Maximum Processing Time: %d' % self.max_time.seconds - print 'Average Processing Time: %f' % self.avg_time - self.elapsed_time = self.latest_time-self.earliest_time - print 'Elapsed Time: %d' % self.elapsed_time.seconds - tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files) - print 'Throughput: %f transactions / minute' % tput - diff --git a/backup/src/boto/services/service.py b/backup/src/boto/services/service.py deleted file mode 100644 index 8ee1a8b..0000000 --- a/backup/src/boto/services/service.py +++ /dev/null @@ -1,161 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto.services.message import ServiceMessage -from boto.services.servicedef import ServiceDef -from boto.pyami.scriptbase import ScriptBase -from boto.utils import get_ts -import time -import os -import mimetypes - - -class Service(ScriptBase): - - # Time required to process a transaction - ProcessingTime = 60 - - def __init__(self, config_file=None, mimetype_files=None): - ScriptBase.__init__(self, config_file) - self.name = self.__class__.__name__ - self.working_dir = boto.config.get('Pyami', 'working_dir') - self.sd = ServiceDef(config_file) - self.retry_count = self.sd.getint('retry_count', 5) - self.loop_delay = self.sd.getint('loop_delay', 30) - self.processing_time = self.sd.getint('processing_time', 60) - self.input_queue = self.sd.get_obj('input_queue') - self.output_queue = self.sd.get_obj('output_queue') - self.output_domain = self.sd.get_obj('output_domain') - if mimetype_files: - mimetypes.init(mimetype_files) - - def split_key(key): - if key.find(';') < 0: - t = (key, '') - else: - key, type = key.split(';') - label, mtype = type.split('=') - t = (key, mtype) - return t - - def read_message(self): - boto.log.info('read_message') - message = self.input_queue.read(self.processing_time) - if message: - boto.log.info(message.get_body()) - key = 'Service-Read' - message[key] = get_ts() - return message - - # retrieve the source file from S3 - def get_file(self, message): - bucket_name = message['Bucket'] - key_name = message['InputKey'] - file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file')) - boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name)) - bucket = boto.lookup('s3', bucket_name) - key = bucket.new_key(key_name) - key.get_contents_to_filename(os.path.join(self.working_dir, file_name)) - return file_name - - # process source file, return list of output files - def process_file(self, in_file_name, msg): - return [] - - # store result file in S3 - def put_file(self, bucket_name, file_path, key_name=None): - boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name)) - bucket = boto.lookup('s3', bucket_name) - key = bucket.new_key(key_name) - key.set_contents_from_filename(file_path) - return key - - def save_results(self, results, input_message, output_message): - output_keys = [] - for file, type in results: - if input_message.has_key('OutputBucket'): - output_bucket = input_message['OutputBucket'] - else: - output_bucket = input_message['Bucket'] - key_name = os.path.split(file)[1] - key = self.put_file(output_bucket, file, key_name) - output_keys.append('%s;type=%s' % (key.name, type)) - output_message['OutputKey'] = ','.join(output_keys) - - # write message to each output queue - def write_message(self, message): - message['Service-Write'] = get_ts() - message['Server'] = self.name - if os.environ.has_key('HOSTNAME'): - message['Host'] = os.environ['HOSTNAME'] - else: - message['Host'] = 'unknown' - message['Instance-ID'] = self.instance_id - if self.output_queue: - boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id) - self.output_queue.write(message) - if self.output_domain: - boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name) - item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']]) - self.output_domain.put_attributes(item_name, message) - - # delete message from input queue - def delete_message(self, message): - boto.log.info('deleting message from %s' % self.input_queue.id) - self.input_queue.delete_message(message) - - # to clean up any files, etc. after each iteration - def cleanup(self): - pass - - def shutdown(self): - on_completion = self.sd.get('on_completion', 'shutdown') - if on_completion == 'shutdown': - if self.instance_id: - time.sleep(60) - c = boto.connect_ec2() - c.terminate_instances([self.instance_id]) - - def main(self, notify=False): - self.notify('Service: %s Starting' % self.name) - empty_reads = 0 - while self.retry_count < 0 or empty_reads < self.retry_count: - try: - input_message = self.read_message() - if input_message: - empty_reads = 0 - output_message = ServiceMessage(None, input_message.get_body()) - input_file = self.get_file(input_message) - results = self.process_file(input_file, output_message) - self.save_results(results, input_message, output_message) - self.write_message(output_message) - self.delete_message(input_message) - self.cleanup() - else: - empty_reads += 1 - time.sleep(self.loop_delay) - except Exception: - boto.log.exception('Service Failed') - empty_reads += 1 - self.notify('Service: %s Shutting Down' % self.name) - self.shutdown() - diff --git a/backup/src/boto/services/servicedef.py b/backup/src/boto/services/servicedef.py deleted file mode 100644 index 1cb01aa..0000000 --- a/backup/src/boto/services/servicedef.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.pyami.config import Config -from boto.services.message import ServiceMessage -import boto - -class ServiceDef(Config): - - def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None): - Config.__init__(self, config_file) - self.aws_access_key_id = aws_access_key_id - self.aws_secret_access_key = aws_secret_access_key - script = Config.get(self, 'Pyami', 'scripts') - if script: - self.name = script.split('.')[-1] - else: - self.name = None - - - def get(self, name, default=None): - return Config.get(self, self.name, name, default) - - def has_option(self, option): - return Config.has_option(self, self.name, option) - - def getint(self, option, default=0): - try: - val = Config.get(self, self.name, option) - val = int(val) - except: - val = int(default) - return val - - def getbool(self, option, default=False): - try: - val = Config.get(self, self.name, option) - if val.lower() == 'true': - val = True - else: - val = False - except: - val = default - return val - - def get_obj(self, name): - """ - Returns the AWS object associated with a given option. - - The heuristics used are a bit lame. If the option name contains - the word 'bucket' it is assumed to be an S3 bucket, if the name - contains the word 'queue' it is assumed to be an SQS queue and - if it contains the word 'domain' it is assumed to be a SimpleDB - domain. If the option name specified does not exist in the - config file or if the AWS object cannot be retrieved this - returns None. - """ - val = self.get(name) - if not val: - return None - if name.find('queue') >= 0: - obj = boto.lookup('sqs', val) - if obj: - obj.set_message_class(ServiceMessage) - elif name.find('bucket') >= 0: - obj = boto.lookup('s3', val) - elif name.find('domain') >= 0: - obj = boto.lookup('sdb', val) - else: - obj = None - return obj - - diff --git a/backup/src/boto/services/sonofmmm.cfg b/backup/src/boto/services/sonofmmm.cfg deleted file mode 100644 index d70d379..0000000 --- a/backup/src/boto/services/sonofmmm.cfg +++ /dev/null @@ -1,43 +0,0 @@ -# -# Your AWS Credentials -# You only need to supply these in this file if you are not using -# the boto tools to start your service -# -#[Credentials] -#aws_access_key_id = -#aws_secret_access_key = - -# -# Fill out this section if you want emails from the service -# when it starts and stops -# -#[Notification] -#smtp_host = -#smtp_user = -#smtp_pass = -#smtp_from = -#smtp_to = - -[Pyami] -scripts = boto.services.sonofmmm.SonOfMMM - -[SonOfMMM] -# id of the AMI to be launched -ami_id = ami-dc799cb5 -# number of times service will read an empty queue before exiting -# a negative value will cause the service to run forever -retry_count = 5 -# seconds to wait after empty queue read before reading again -loop_delay = 10 -# average time it takes to process a transaction -# controls invisibility timeout of messages -processing_time = 60 -ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s -output_mimetype = video/quicktime -output_ext = .mov -input_bucket = -output_bucket = -output_domain = -output_queue = -input_queue = - diff --git a/backup/src/boto/services/sonofmmm.py b/backup/src/boto/services/sonofmmm.py deleted file mode 100644 index acb7e61..0000000 --- a/backup/src/boto/services/sonofmmm.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import boto -from boto.services.service import Service -from boto.services.message import ServiceMessage -import os -import mimetypes - -class SonOfMMM(Service): - - def __init__(self, config_file=None): - Service.__init__(self, config_file) - self.log_file = '%s.log' % self.instance_id - self.log_path = os.path.join(self.working_dir, self.log_file) - boto.set_file_logger(self.name, self.log_path) - if self.sd.has_option('ffmpeg_args'): - self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') - else: - self.command = '/usr/local/bin/ffmpeg -y -i %s %s' - self.output_mimetype = self.sd.get('output_mimetype') - if self.sd.has_option('output_ext'): - self.output_ext = self.sd.get('output_ext') - else: - self.output_ext = mimetypes.guess_extension(self.output_mimetype) - self.output_bucket = self.sd.get_obj('output_bucket') - self.input_bucket = self.sd.get_obj('input_bucket') - # check to see if there are any messages queue - # if not, create messages for all files in input_bucket - m = self.input_queue.read(1) - if not m: - self.queue_files() - - def queue_files(self): - boto.log.info('Queueing files from %s' % self.input_bucket.name) - for key in self.input_bucket: - boto.log.info('Queueing %s' % key.name) - m = ServiceMessage() - if self.output_bucket: - d = {'OutputBucket' : self.output_bucket.name} - else: - d = None - m.for_key(key, d) - self.input_queue.write(m) - - def process_file(self, in_file_name, msg): - base, ext = os.path.splitext(in_file_name) - out_file_name = os.path.join(self.working_dir, - base+self.output_ext) - command = self.command % (in_file_name, out_file_name) - boto.log.info('running:\n%s' % command) - status = self.run(command) - if status == 0: - return [(out_file_name, self.output_mimetype)] - else: - return [] - - def shutdown(self): - if os.path.isfile(self.log_path): - if self.output_bucket: - key = self.output_bucket.new_key(self.log_file) - key.set_contents_from_filename(self.log_path) - Service.shutdown(self) diff --git a/backup/src/boto/services/submit.py b/backup/src/boto/services/submit.py deleted file mode 100644 index 89c439c..0000000 --- a/backup/src/boto/services/submit.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import time -import os - - -class Submitter: - - def __init__(self, sd): - self.sd = sd - self.input_bucket = self.sd.get_obj('input_bucket') - self.output_bucket = self.sd.get_obj('output_bucket') - self.output_domain = self.sd.get_obj('output_domain') - self.queue = self.sd.get_obj('input_queue') - - def get_key_name(self, fullpath, prefix): - key_name = fullpath[len(prefix):] - l = key_name.split(os.sep) - return '/'.join(l) - - def write_message(self, key, metadata): - if self.queue: - m = self.queue.new_message() - m.for_key(key, metadata) - if self.output_bucket: - m['OutputBucket'] = self.output_bucket.name - self.queue.write(m) - - def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'): - if not metadata: - metadata = {} - key_name = self.get_key_name(path, prefix) - k = self.input_bucket.new_key(key_name) - k.update_metadata(metadata) - k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb) - self.write_message(k, metadata) - - def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'): - path = os.path.expanduser(path) - path = os.path.expandvars(path) - path = os.path.abspath(path) - total = 0 - metadata = {} - if tags: - metadata['Tags'] = tags - l = [] - for t in time.gmtime(): - l.append(str(t)) - metadata['Batch'] = '_'.join(l) - if self.output_domain: - self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'}) - if os.path.isdir(path): - for root, dirs, files in os.walk(path): - if ignore_dirs: - for ignore in ignore_dirs: - if ignore in dirs: - dirs.remove(ignore) - for file in files: - fullpath = os.path.join(root, file) - if status: - print 'Submitting %s' % fullpath - self.submit_file(fullpath, metadata, cb, num_cb, prefix) - total += 1 - elif os.path.isfile(path): - self.submit_file(path, metadata, cb, num_cb) - total += 1 - else: - print 'problem with %s' % path - return (metadata['Batch'], total) diff --git a/backup/src/boto/ses/__init__.py b/backup/src/boto/ses/__init__.py deleted file mode 100644 index 167080b..0000000 --- a/backup/src/boto/ses/__init__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2011 Harry Marr http://hmarr.com/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from connection import SESConnection - diff --git a/backup/src/boto/ses/connection.py b/backup/src/boto/ses/connection.py deleted file mode 100644 index 8268e4a..0000000 --- a/backup/src/boto/ses/connection.py +++ /dev/null @@ -1,270 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2011 Harry Marr http://hmarr.com/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.connection import AWSAuthConnection -from boto.exception import BotoServerError -import boto -import boto.jsonresponse - -import urllib -import base64 - - -class SESConnection(AWSAuthConnection): - - ResponseError = BotoServerError - DefaultHost = 'email.us-east-1.amazonaws.com' - APIVersion = '2010-12-01' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - port=None, proxy=None, proxy_port=None, - host=DefaultHost, debug=0): - AWSAuthConnection.__init__(self, host, aws_access_key_id, - aws_secret_access_key, True, port, proxy, - proxy_port, debug=debug) - - def _required_auth_capability(self): - return ['ses'] - - def _build_list_params(self, params, items, label): - """Add an AWS API-compatible parameter list to a dictionary. - - :type params: dict - :param params: The parameter dictionary - - :type items: list - :param items: Items to be included in the list - - :type label: string - :param label: The parameter list's name - """ - if isinstance(items, str): - items = [items] - for i in range(1, len(items) + 1): - params['%s.%d' % (label, i)] = items[i - 1] - - - def _make_request(self, action, params=None): - """Make a call to the SES API. - - :type action: string - :param action: The API method to use (e.g. SendRawEmail) - - :type params: dict - :param params: Parameters that will be sent as POST data with the API - call. - """ - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - params = params or {} - params['Action'] = action - response = super(SESConnection, self).make_request( - 'POST', - '/', - headers=headers, - data=urllib.urlencode(params) - ) - body = response.read() - if response.status == 200: - list_markers = ('VerifiedEmailAddresses', 'SendDataPoints') - e = boto.jsonresponse.Element(list_marker=list_markers) - h = boto.jsonresponse.XmlHandler(e, None) - h.parse(body) - return e - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - - def send_email(self, source, subject, body, to_addresses, cc_addresses=None, - bcc_addresses=None, format='text', reply_addresses=None, - return_path=None): - """Composes an email message based on input data, and then immediately - queues the message for sending. - - :type source: string - :param source: The sender's email address. - - :type subject: string - :param subject: The subject of the message: A short summary of the - content, which will appear in the recipient's inbox. - - :type body: string - :param body: The message body. - - :type to_addresses: list of strings or string - :param to_addresses: The To: field(s) of the message. - - :type cc_addresses: list of strings or string - :param cc_addresses: The CC: field(s) of the message. - - :type bcc_addresses: list of strings or string - :param bcc_addresses: The BCC: field(s) of the message. - - :type format: string - :param format: The format of the message's body, must be either "text" - or "html". - - :type reply_addresses: list of strings or string - :param reply_addresses: The reply-to email address(es) for the - message. If the recipient replies to the - message, each reply-to address will - receive the reply. - - :type return_path: string - :param return_path: The email address to which bounce notifications are - to be forwarded. If the message cannot be delivered - to the recipient, then an error message will be - returned from the recipient's ISP; this message will - then be forwarded to the email address specified by - the ReturnPath parameter. - - """ - params = { - 'Source': source, - 'Message.Subject.Data': subject, - } - - if return_path: - params['ReturnPath'] = return_path - - format = format.lower().strip() - if format == 'html': - params['Message.Body.Html.Data'] = body - elif format == 'text': - params['Message.Body.Text.Data'] = body - else: - raise ValueError("'format' argument must be 'text' or 'html'") - - self._build_list_params(params, to_addresses, - 'Destination.ToAddresses.member') - if cc_addresses: - self._build_list_params(params, cc_addresses, - 'Destination.CcAddresses.member') - - if bcc_addresses: - self._build_list_params(params, bcc_addresses, - 'Destination.BccAddresses.member') - - if reply_addresses: - self._build_list_params(params, reply_addresses, - 'ReplyToAddresses.member') - - return self._make_request('SendEmail', params) - - def send_raw_email(self, source, raw_message, destinations=None): - """Sends an email message, with header and content specified by the - client. The SendRawEmail action is useful for sending multipart MIME - emails, with attachments or inline content. The raw text of the message - must comply with Internet email standards; otherwise, the message - cannot be sent. - - :type source: string - :param source: The sender's email address. - - :type raw_message: string - :param raw_message: The raw text of the message. The client is - responsible for ensuring the following: - - - Message must contain a header and a body, separated by a blank line. - - All required header fields must be present. - - Each part of a multipart MIME message must be formatted properly. - - MIME content types must be among those supported by Amazon SES. - Refer to the Amazon SES Developer Guide for more details. - - Content must be base64-encoded, if MIME requires it. - - :type destinations: list of strings or string - :param destinations: A list of destinations for the message. - - """ - params = { - 'Source': source, - 'RawMessage.Data': base64.b64encode(raw_message), - } - - self._build_list_params(params, destinations, - 'Destinations.member') - - return self._make_request('SendRawEmail', params) - - def list_verified_email_addresses(self): - """Fetch a list of the email addresses that have been verified. - - :rtype: dict - :returns: A ListVerifiedEmailAddressesResponse structure. Note that - keys must be unicode strings. - """ - return self._make_request('ListVerifiedEmailAddresses') - - def get_send_quota(self): - """Fetches the user's current activity limits. - - :rtype: dict - :returns: A GetSendQuotaResponse structure. Note that keys must be - unicode strings. - """ - return self._make_request('GetSendQuota') - - def get_send_statistics(self): - """Fetches the user's sending statistics. The result is a list of data - points, representing the last two weeks of sending activity. - - Each data point in the list contains statistics for a 15-minute - interval. - - :rtype: dict - :returns: A GetSendStatisticsResponse structure. Note that keys must be - unicode strings. - """ - return self._make_request('GetSendStatistics') - - def delete_verified_email_address(self, email_address): - """Deletes the specified email address from the list of verified - addresses. - - :type email_adddress: string - :param email_address: The email address to be removed from the list of - verified addreses. - - :rtype: dict - :returns: A DeleteVerifiedEmailAddressResponse structure. Note that - keys must be unicode strings. - """ - return self._make_request('DeleteVerifiedEmailAddress', { - 'EmailAddress': email_address, - }) - - def verify_email_address(self, email_address): - """Verifies an email address. This action causes a confirmation email - message to be sent to the specified address. - - :type email_adddress: string - :param email_address: The email address to be verified. - - :rtype: dict - :returns: A VerifyEmailAddressResponse structure. Note that keys must - be unicode strings. - """ - return self._make_request('VerifyEmailAddress', { - 'EmailAddress': email_address, - }) - diff --git a/backup/src/boto/sns/__init__.py b/backup/src/boto/sns/__init__.py deleted file mode 100644 index 9c5a7d7..0000000 --- a/backup/src/boto/sns/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# this is here for backward compatibility -# originally, the SNSConnection class was defined here -from connection import SNSConnection diff --git a/backup/src/boto/sns/connection.py b/backup/src/boto/sns/connection.py deleted file mode 100644 index 2a49adb..0000000 --- a/backup/src/boto/sns/connection.py +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.connection import AWSQueryConnection -from boto.sdb.regioninfo import SDBRegionInfo -import boto -import uuid -try: - import json -except ImportError: - import simplejson as json - -#boto.set_stream_logger('sns') - -class SNSConnection(AWSQueryConnection): - - DefaultRegionName = 'us-east-1' - DefaultRegionEndpoint = 'sns.us-east-1.amazonaws.com' - APIVersion = '2010-03-31' - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/', converter=None): - if not region: - region = SDBRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - self.region.endpoint, debug, https_connection_factory, path) - - def _required_auth_capability(self): - return ['sns'] - - def get_all_topics(self, next_token=None): - """ - :type next_token: string - :param next_token: Token returned by the previous call to - this method. - - """ - params = {'ContentType' : 'JSON'} - if next_token: - params['NextToken'] = next_token - response = self.make_request('ListTopics', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_topic_attributes(self, topic): - """ - Get attributes of a Topic - - :type topic: string - :param topic: The ARN of the topic. - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic} - response = self.make_request('GetTopicAttributes', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def add_permission(self, topic, label, account_ids, actions): - """ - Adds a statement to a topic's access control policy, granting - access for the specified AWS accounts to the specified actions. - - :type topic: string - :param topic: The ARN of the topic. - - :type label: string - :param label: A unique identifier for the new policy statement. - - :type account_ids: list of strings - :param account_ids: The AWS account ids of the users who will be - give access to the specified actions. - - :type actions: list of strings - :param actions: The actions you want to allow for each of the - specified principal(s). - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic, - 'Label' : label} - self.build_list_params(params, account_ids, 'AWSAccountId') - self.build_list_params(params, actions, 'ActionName') - response = self.make_request('AddPermission', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def remove_permission(self, topic, label): - """ - Removes a statement from a topic's access control policy. - - :type topic: string - :param topic: The ARN of the topic. - - :type label: string - :param label: A unique identifier for the policy statement - to be removed. - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic, - 'Label' : label} - response = self.make_request('RemovePermission', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def create_topic(self, topic): - """ - Create a new Topic. - - :type topic: string - :param topic: The name of the new topic. - - """ - params = {'ContentType' : 'JSON', - 'Name' : topic} - response = self.make_request('CreateTopic', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def delete_topic(self, topic): - """ - Delete an existing topic - - :type topic: string - :param topic: The ARN of the topic - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic} - response = self.make_request('DeleteTopic', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - - - def publish(self, topic, message, subject=None): - """ - Get properties of a Topic - - :type topic: string - :param topic: The ARN of the new topic. - - :type message: string - :param message: The message you want to send to the topic. - Messages must be UTF-8 encoded strings and - be at most 4KB in size. - - :type subject: string - :param subject: Optional parameter to be used as the "Subject" - line of the email notifications. - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic, - 'Message' : message} - if subject: - params['Subject'] = subject - response = self.make_request('Publish', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def subscribe(self, topic, protocol, endpoint): - """ - Subscribe to a Topic. - - :type topic: string - :param topic: The name of the new topic. - - :type protocol: string - :param protocol: The protocol used to communicate with - the subscriber. Current choices are: - email|email-json|http|https|sqs - - :type endpoint: string - :param endpoint: The location of the endpoint for - the subscriber. - * For email, this would be a valid email address - * For email-json, this would be a valid email address - * For http, this would be a URL beginning with http - * For https, this would be a URL beginning with https - * For sqs, this would be the ARN of an SQS Queue - - :rtype: :class:`boto.sdb.domain.Domain` object - :return: The newly created domain - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic, - 'Protocol' : protocol, - 'Endpoint' : endpoint} - response = self.make_request('Subscribe', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def subscribe_sqs_queue(self, topic, queue): - """ - Subscribe an SQS queue to a topic. - - This is convenience method that handles most of the complexity involved - in using ans SQS queue as an endpoint for an SNS topic. To achieve this - the following operations are performed: - - * The correct ARN is constructed for the SQS queue and that ARN is - then subscribed to the topic. - * A JSON policy document is contructed that grants permission to - the SNS topic to send messages to the SQS queue. - * This JSON policy is then associated with the SQS queue using - the queue's set_attribute method. If the queue already has - a policy associated with it, this process will add a Statement to - that policy. If no policy exists, a new policy will be created. - - :type topic: string - :param topic: The name of the new topic. - - :type queue: A boto Queue object - :param queue: The queue you wish to subscribe to the SNS Topic. - """ - t = queue.id.split('/') - q_arn = 'arn:aws:sqs:%s:%s:%s' % (queue.connection.region.name, - t[1], t[2]) - resp = self.subscribe(topic, 'sqs', q_arn) - policy = queue.get_attributes('Policy') - if 'Version' not in policy: - policy['Version'] = '2008-10-17' - if 'Statement' not in policy: - policy['Statement'] = [] - statement = {'Action' : 'SQS:SendMessage', - 'Effect' : 'Allow', - 'Principal' : {'AWS' : '*'}, - 'Resource' : q_arn, - 'Sid' : str(uuid.uuid4()), - 'Condition' : {'StringLike' : {'aws:SourceArn' : topic}}} - policy['Statement'].append(statement) - queue.set_attribute('Policy', json.dumps(policy)) - return resp - - def confirm_subscription(self, topic, token, - authenticate_on_unsubscribe=False): - """ - Get properties of a Topic - - :type topic: string - :param topic: The ARN of the new topic. - - :type token: string - :param token: Short-lived token sent to and endpoint during - the Subscribe operation. - - :type authenticate_on_unsubscribe: bool - :param authenticate_on_unsubscribe: Optional parameter indicating - that you wish to disable - unauthenticated unsubscription - of the subscription. - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic, - 'Token' : token} - if authenticate_on_unsubscribe: - params['AuthenticateOnUnsubscribe'] = 'true' - response = self.make_request('ConfirmSubscription', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def unsubscribe(self, subscription): - """ - Allows endpoint owner to delete subscription. - Confirmation message will be delivered. - - :type subscription: string - :param subscription: The ARN of the subscription to be deleted. - - """ - params = {'ContentType' : 'JSON', - 'SubscriptionArn' : subscription} - response = self.make_request('Unsubscribe', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_all_subscriptions(self, next_token=None): - """ - Get list of all subscriptions. - - :type next_token: string - :param next_token: Token returned by the previous call to - this method. - - """ - params = {'ContentType' : 'JSON'} - if next_token: - params['NextToken'] = next_token - response = self.make_request('ListSubscriptions', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - - def get_all_subscriptions_by_topic(self, topic, next_token=None): - """ - Get list of all subscriptions to a specific topic. - - :type topic: string - :param topic: The ARN of the topic for which you wish to - find subscriptions. - - :type next_token: string - :param next_token: Token returned by the previous call to - this method. - - """ - params = {'ContentType' : 'JSON', - 'TopicArn' : topic} - if next_token: - params['NextToken'] = next_token - response = self.make_request('ListSubscriptions', params, '/', 'GET') - body = response.read() - if response.status == 200: - return json.loads(body) - else: - boto.log.error('%s %s' % (response.status, response.reason)) - boto.log.error('%s' % body) - raise self.ResponseError(response.status, response.reason, body) - diff --git a/backup/src/boto/sqs/__init__.py b/backup/src/boto/sqs/__init__.py deleted file mode 100644 index 463c42c..0000000 --- a/backup/src/boto/sqs/__init__.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from regioninfo import SQSRegionInfo - -def regions(): - """ - Get all available regions for the SQS service. - - :rtype: list - :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` - """ - return [SQSRegionInfo(name='us-east-1', - endpoint='queue.amazonaws.com'), - SQSRegionInfo(name='eu-west-1', - endpoint='eu-west-1.queue.amazonaws.com'), - SQSRegionInfo(name='us-west-1', - endpoint='us-west-1.queue.amazonaws.com'), - SQSRegionInfo(name='ap-southeast-1', - endpoint='ap-southeast-1.queue.amazonaws.com') - ] - -def connect_to_region(region_name): - for region in regions(): - if region.name == region_name: - return region.connect() - return None diff --git a/backup/src/boto/sqs/attributes.py b/backup/src/boto/sqs/attributes.py deleted file mode 100644 index 26c7204..0000000 --- a/backup/src/boto/sqs/attributes.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an SQS Attribute Name/Value set -""" - -class Attributes(dict): - - def __init__(self, parent): - self.parent = parent - self.current_key = None - self.current_value = None - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'Attribute': - self[self.current_key] = self.current_value - elif name == 'Name': - self.current_key = value - elif name == 'Value': - self.current_value = value - else: - setattr(self, name, value) - - diff --git a/backup/src/boto/sqs/connection.py b/backup/src/boto/sqs/connection.py deleted file mode 100644 index 240fc72..0000000 --- a/backup/src/boto/sqs/connection.py +++ /dev/null @@ -1,288 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.connection import AWSQueryConnection -from boto.sqs.regioninfo import SQSRegionInfo -from boto.sqs.queue import Queue -from boto.sqs.message import Message -from boto.sqs.attributes import Attributes -from boto.exception import SQSError - - -class SQSConnection(AWSQueryConnection): - """ - A Connection to the SQS Service. - """ - DefaultRegionName = 'us-east-1' - DefaultRegionEndpoint = 'queue.amazonaws.com' - APIVersion = '2009-02-01' - DefaultContentType = 'text/plain' - ResponseError = SQSError - - def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, - is_secure=True, port=None, proxy=None, proxy_port=None, - proxy_user=None, proxy_pass=None, debug=0, - https_connection_factory=None, region=None, path='/'): - if not region: - region = SQSRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) - self.region = region - AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, - is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, - self.region.endpoint, debug, https_connection_factory, path) - - def _required_auth_capability(self): - return ['sqs'] - - def create_queue(self, queue_name, visibility_timeout=None): - """ - Create an SQS Queue. - - :type queue_name: str or unicode - :param queue_name: The name of the new queue. Names are scoped to an account and need to - be unique within that account. Calling this method on an existing - queue name will not return an error from SQS unless the value for - visibility_timeout is different than the value of the existing queue - of that name. This is still an expensive operation, though, and not - the preferred way to check for the existence of a queue. See the - :func:`boto.sqs.connection.SQSConnection.lookup` method. - - :type visibility_timeout: int - :param visibility_timeout: The default visibility timeout for all messages written in the - queue. This can be overridden on a per-message. - - :rtype: :class:`boto.sqs.queue.Queue` - :return: The newly created queue. - - """ - params = {'QueueName': queue_name} - if visibility_timeout: - params['DefaultVisibilityTimeout'] = '%d' % (visibility_timeout,) - return self.get_object('CreateQueue', params, Queue) - - def delete_queue(self, queue, force_deletion=False): - """ - Delete an SQS Queue. - - :type queue: A Queue object - :param queue: The SQS queue to be deleted - - :type force_deletion: Boolean - :param force_deletion: Normally, SQS will not delete a queue that contains messages. - However, if the force_deletion argument is True, the - queue will be deleted regardless of whether there are messages in - the queue or not. USE WITH CAUTION. This will delete all - messages in the queue as well. - - :rtype: bool - :return: True if the command succeeded, False otherwise - """ - return self.get_status('DeleteQueue', None, queue.id) - - def get_queue_attributes(self, queue, attribute='All'): - """ - Gets one or all attributes of a Queue - - :type queue: A Queue object - :param queue: The SQS queue to be deleted - - :type attribute: str - :type attribute: The specific attribute requested. If not supplied, the default - is to return all attributes. Valid attributes are: - ApproximateNumberOfMessages, - ApproximateNumberOfMessagesNotVisible, - VisibilityTimeout, - CreatedTimestamp, - LastModifiedTimestamp, - Policy - - :rtype: :class:`boto.sqs.attributes.Attributes` - :return: An Attributes object containing request value(s). - """ - params = {'AttributeName' : attribute} - return self.get_object('GetQueueAttributes', params, Attributes, queue.id) - - def set_queue_attribute(self, queue, attribute, value): - params = {'Attribute.Name' : attribute, 'Attribute.Value' : value} - return self.get_status('SetQueueAttributes', params, queue.id) - - def receive_message(self, queue, number_messages=1, visibility_timeout=None, - attributes=None): - """ - Read messages from an SQS Queue. - - :type queue: A Queue object - :param queue: The Queue from which messages are read. - - :type number_messages: int - :param number_messages: The maximum number of messages to read (default=1) - - :type visibility_timeout: int - :param visibility_timeout: The number of seconds the message should remain invisible - to other queue readers (default=None which uses the Queues default) - - :type attributes: str - :param attributes: The name of additional attribute to return with response - or All if you want all attributes. The default is to - return no additional attributes. Valid values: - All - SenderId - SentTimestamp - ApproximateReceiveCount - ApproximateFirstReceiveTimestamp - - :rtype: list - :return: A list of :class:`boto.sqs.message.Message` objects. - """ - params = {'MaxNumberOfMessages' : number_messages} - if visibility_timeout: - params['VisibilityTimeout'] = visibility_timeout - if attributes: - self.build_list_params(params, attributes, 'AttributeName') - return self.get_list('ReceiveMessage', params, [('Message', queue.message_class)], - queue.id, queue) - - def delete_message(self, queue, message): - """ - Delete a message from a queue. - - :type queue: A :class:`boto.sqs.queue.Queue` object - :param queue: The Queue from which messages are read. - - :type message: A :class:`boto.sqs.message.Message` object - :param message: The Message to be deleted - - :rtype: bool - :return: True if successful, False otherwise. - """ - params = {'ReceiptHandle' : message.receipt_handle} - return self.get_status('DeleteMessage', params, queue.id) - - def delete_message_from_handle(self, queue, receipt_handle): - """ - Delete a message from a queue, given a receipt handle. - - :type queue: A :class:`boto.sqs.queue.Queue` object - :param queue: The Queue from which messages are read. - - :type receipt_handle: str - :param receipt_handle: The receipt handle for the message - - :rtype: bool - :return: True if successful, False otherwise. - """ - params = {'ReceiptHandle' : receipt_handle} - return self.get_status('DeleteMessage', params, queue.id) - - def send_message(self, queue, message_content): - params = {'MessageBody' : message_content} - return self.get_object('SendMessage', params, Message, queue.id, verb='POST') - - def change_message_visibility(self, queue, receipt_handle, visibility_timeout): - """ - Extends the read lock timeout for the specified message from the specified queue - to the specified value. - - :type queue: A :class:`boto.sqs.queue.Queue` object - :param queue: The Queue from which messages are read. - - :type receipt_handle: str - :param queue: The receipt handle associated with the message whose - visibility timeout will be changed. - - :type visibility_timeout: int - :param visibility_timeout: The new value of the message's visibility timeout - in seconds. - """ - params = {'ReceiptHandle' : receipt_handle, - 'VisibilityTimeout' : visibility_timeout} - return self.get_status('ChangeMessageVisibility', params, queue.id) - - def get_all_queues(self, prefix=''): - params = {} - if prefix: - params['QueueNamePrefix'] = prefix - return self.get_list('ListQueues', params, [('QueueUrl', Queue)]) - - def get_queue(self, queue_name): - rs = self.get_all_queues(queue_name) - for q in rs: - if q.url.endswith(queue_name): - return q - return None - - lookup = get_queue - - # - # Permissions methods - # - - def add_permission(self, queue, label, aws_account_id, action_name): - """ - Add a permission to a queue. - - :type queue: :class:`boto.sqs.queue.Queue` - :param queue: The queue object - - :type label: str or unicode - :param label: A unique identification of the permission you are setting. - Maximum of 80 characters ``[0-9a-zA-Z_-]`` - Example, AliceSendMessage - - :type aws_account_id: str or unicode - :param principal_id: The AWS account number of the principal who will be given - permission. The principal must have an AWS account, but - does not need to be signed up for Amazon SQS. For information - about locating the AWS account identification. - - :type action_name: str or unicode - :param action_name: The action. Valid choices are: - \*|SendMessage|ReceiveMessage|DeleteMessage| - ChangeMessageVisibility|GetQueueAttributes - - :rtype: bool - :return: True if successful, False otherwise. - - """ - params = {'Label': label, - 'AWSAccountId' : aws_account_id, - 'ActionName' : action_name} - return self.get_status('AddPermission', params, queue.id) - - def remove_permission(self, queue, label): - """ - Remove a permission from a queue. - - :type queue: :class:`boto.sqs.queue.Queue` - :param queue: The queue object - - :type label: str or unicode - :param label: The unique label associated with the permission being removed. - - :rtype: bool - :return: True if successful, False otherwise. - """ - params = {'Label': label} - return self.get_status('RemovePermission', params, queue.id) - - - - - diff --git a/backup/src/boto/sqs/jsonmessage.py b/backup/src/boto/sqs/jsonmessage.py deleted file mode 100644 index 24a3be2..0000000 --- a/backup/src/boto/sqs/jsonmessage.py +++ /dev/null @@ -1,45 +0,0 @@ -# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -from boto.sqs.message import MHMessage -from boto.exception import SQSDecodeError -import base64 -try: - import json -except ImportError: - import simplejson as json - -class JSONMessage(MHMessage): - """ - Acts like a dictionary but encodes it's data as a Base64 encoded JSON payload. - """ - - def decode(self, value): - try: - value = base64.b64decode(value) - value = json.loads(value) - except: - raise SQSDecodeError('Unable to decode message', self) - return value - - def encode(self, value): - value = json.dumps(value) - return base64.b64encode(value) diff --git a/backup/src/boto/sqs/message.py b/backup/src/boto/sqs/message.py deleted file mode 100644 index 8fabd47..0000000 --- a/backup/src/boto/sqs/message.py +++ /dev/null @@ -1,251 +0,0 @@ -# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -SQS Message - -A Message represents the data stored in an SQS queue. The rules for what is allowed within an SQS -Message are here: - - http://docs.amazonwebservices.com/AWSSimpleQueueService/2008-01-01/SQSDeveloperGuide/Query_QuerySendMessage.html - -So, at it's simplest level a Message just needs to allow a developer to store bytes in it and get the bytes -back out. However, to allow messages to have richer semantics, the Message class must support the -following interfaces: - -The constructor for the Message class must accept a keyword parameter "queue" which is an instance of a -boto Queue object and represents the queue that the message will be stored in. The default value for -this parameter is None. - -The constructor for the Message class must accept a keyword parameter "body" which represents the -content or body of the message. The format of this parameter will depend on the behavior of the -particular Message subclass. For example, if the Message subclass provides dictionary-like behavior to the -user the body passed to the constructor should be a dict-like object that can be used to populate -the initial state of the message. - -The Message class must provide an encode method that accepts a value of the same type as the body -parameter of the constructor and returns a string of characters that are able to be stored in an -SQS message body (see rules above). - -The Message class must provide a decode method that accepts a string of characters that can be -stored (and probably were stored!) in an SQS message and return an object of a type that is consistent -with the "body" parameter accepted on the class constructor. - -The Message class must provide a __len__ method that will return the size of the encoded message -that would be stored in SQS based on the current state of the Message object. - -The Message class must provide a get_body method that will return the body of the message in the -same format accepted in the constructor of the class. - -The Message class must provide a set_body method that accepts a message body in the same format -accepted by the constructor of the class. This method should alter to the internal state of the -Message object to reflect the state represented in the message body parameter. - -The Message class must provide a get_body_encoded method that returns the current body of the message -in the format in which it would be stored in SQS. -""" - -import base64 -import StringIO -from boto.sqs.attributes import Attributes -from boto.exception import SQSDecodeError - -class RawMessage: - """ - Base class for SQS messages. RawMessage does not encode the message - in any way. Whatever you store in the body of the message is what - will be written to SQS and whatever is returned from SQS is stored - directly into the body of the message. - """ - - def __init__(self, queue=None, body=''): - self.queue = queue - self.set_body(body) - self.id = None - self.receipt_handle = None - self.md5 = None - self.attributes = Attributes(self) - - def __len__(self): - return len(self.encode(self._body)) - - def startElement(self, name, attrs, connection): - if name == 'Attribute': - return self.attributes - return None - - def endElement(self, name, value, connection): - if name == 'Body': - self.set_body(self.decode(value)) - elif name == 'MessageId': - self.id = value - elif name == 'ReceiptHandle': - self.receipt_handle = value - elif name == 'MD5OfMessageBody': - self.md5 = value - else: - setattr(self, name, value) - - def encode(self, value): - """Transform body object into serialized byte array format.""" - return value - - def decode(self, value): - """Transform seralized byte array into any object.""" - return value - - def set_body(self, body): - """Override the current body for this object, using decoded format.""" - self._body = body - - def get_body(self): - return self._body - - def get_body_encoded(self): - """ - This method is really a semi-private method used by the Queue.write - method when writing the contents of the message to SQS. - You probably shouldn't need to call this method in the normal course of events. - """ - return self.encode(self.get_body()) - - def delete(self): - if self.queue: - return self.queue.delete_message(self) - - def change_visibility(self, visibility_timeout): - if self.queue: - self.queue.connection.change_message_visibility(self.queue, - self.receipt_handle, - visibility_timeout) - -class Message(RawMessage): - """ - The default Message class used for SQS queues. This class automatically - encodes/decodes the message body using Base64 encoding to avoid any - illegal characters in the message body. See: - - http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 - - for details on why this is a good idea. The encode/decode is meant to - be transparent to the end-user. - """ - - def encode(self, value): - return base64.b64encode(value) - - def decode(self, value): - try: - value = base64.b64decode(value) - except: - raise SQSDecodeError('Unable to decode message', self) - return value - -class MHMessage(Message): - """ - The MHMessage class provides a message that provides RFC821-like - headers like this: - - HeaderName: HeaderValue - - The encoding/decoding of this is handled automatically and after - the message body has been read, the message instance can be treated - like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. - """ - - def __init__(self, queue=None, body=None, xml_attrs=None): - if body == None or body == '': - body = {} - Message.__init__(self, queue, body) - - def decode(self, value): - try: - msg = {} - fp = StringIO.StringIO(value) - line = fp.readline() - while line: - delim = line.find(':') - key = line[0:delim] - value = line[delim+1:].strip() - msg[key.strip()] = value.strip() - line = fp.readline() - except: - raise SQSDecodeError('Unable to decode message', self) - return msg - - def encode(self, value): - s = '' - for item in value.items(): - s = s + '%s: %s\n' % (item[0], item[1]) - return s - - def __getitem__(self, key): - if self._body.has_key(key): - return self._body[key] - else: - raise KeyError(key) - - def __setitem__(self, key, value): - self._body[key] = value - self.set_body(self._body) - - def keys(self): - return self._body.keys() - - def values(self): - return self._body.values() - - def items(self): - return self._body.items() - - def has_key(self, key): - return self._body.has_key(key) - - def update(self, d): - self._body.update(d) - self.set_body(self._body) - - def get(self, key, default=None): - return self._body.get(key, default) - -class EncodedMHMessage(MHMessage): - """ - The EncodedMHMessage class provides a message that provides RFC821-like - headers like this: - - HeaderName: HeaderValue - - This variation encodes/decodes the body of the message in base64 automatically. - The message instance can be treated like a mapping object, - i.e. m['HeaderName'] would return 'HeaderValue'. - """ - - def decode(self, value): - try: - value = base64.b64decode(value) - except: - raise SQSDecodeError('Unable to decode message', self) - return MHMessage.decode(self, value) - - def encode(self, value): - value = MHMessage.encode(self, value) - return base64.b64encode(value) - diff --git a/backup/src/boto/sqs/queue.py b/backup/src/boto/sqs/queue.py deleted file mode 100644 index 9965e43..0000000 --- a/backup/src/boto/sqs/queue.py +++ /dev/null @@ -1,414 +0,0 @@ -# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents an SQS Queue -""" - -import urlparse -from boto.sqs.message import Message - - -class Queue: - - def __init__(self, connection=None, url=None, message_class=Message): - self.connection = connection - self.url = url - self.message_class = message_class - self.visibility_timeout = None - - def _id(self): - if self.url: - val = urlparse.urlparse(self.url)[2] - else: - val = self.url - return val - id = property(_id) - - def _name(self): - if self.url: - val = urlparse.urlparse(self.url)[2].split('/')[2] - else: - val = self.url - return val - name = property(_name) - - def startElement(self, name, attrs, connection): - return None - - def endElement(self, name, value, connection): - if name == 'QueueUrl': - self.url = value - elif name == 'VisibilityTimeout': - self.visibility_timeout = int(value) - else: - setattr(self, name, value) - - def set_message_class(self, message_class): - """ - Set the message class that should be used when instantiating messages read - from the queue. By default, the class boto.sqs.message.Message is used but - this can be overriden with any class that behaves like a message. - - :type message_class: Message-like class - :param message_class: The new Message class - """ - self.message_class = message_class - - def get_attributes(self, attributes='All'): - """ - Retrieves attributes about this queue object and returns - them in an Attribute instance (subclass of a Dictionary). - - :type attributes: string - :param attributes: String containing one of: - ApproximateNumberOfMessages, - ApproximateNumberOfMessagesNotVisible, - VisibilityTimeout, - CreatedTimestamp, - LastModifiedTimestamp, - Policy - :rtype: Attribute object - :return: An Attribute object which is a mapping type holding the - requested name/value pairs - """ - return self.connection.get_queue_attributes(self, attributes) - - def set_attribute(self, attribute, value): - """ - Set a new value for an attribute of the Queue. - - :type attribute: String - :param attribute: The name of the attribute you want to set. The - only valid value at this time is: VisibilityTimeout - :type value: int - :param value: The new value for the attribute. - For VisibilityTimeout the value must be an - integer number of seconds from 0 to 86400. - - :rtype: bool - :return: True if successful, otherwise False. - """ - return self.connection.set_queue_attribute(self, attribute, value) - - def get_timeout(self): - """ - Get the visibility timeout for the queue. - - :rtype: int - :return: The number of seconds as an integer. - """ - a = self.get_attributes('VisibilityTimeout') - return int(a['VisibilityTimeout']) - - def set_timeout(self, visibility_timeout): - """ - Set the visibility timeout for the queue. - - :type visibility_timeout: int - :param visibility_timeout: The desired timeout in seconds - """ - retval = self.set_attribute('VisibilityTimeout', visibility_timeout) - if retval: - self.visibility_timeout = visibility_timeout - return retval - - def add_permission(self, label, aws_account_id, action_name): - """ - Add a permission to a queue. - - :type label: str or unicode - :param label: A unique identification of the permission you are setting. - Maximum of 80 characters ``[0-9a-zA-Z_-]`` - Example, AliceSendMessage - - :type aws_account_id: str or unicode - :param principal_id: The AWS account number of the principal who will be given - permission. The principal must have an AWS account, but - does not need to be signed up for Amazon SQS. For information - about locating the AWS account identification. - - :type action_name: str or unicode - :param action_name: The action. Valid choices are: - \*|SendMessage|ReceiveMessage|DeleteMessage| - ChangeMessageVisibility|GetQueueAttributes - - :rtype: bool - :return: True if successful, False otherwise. - - """ - return self.connection.add_permission(self, label, aws_account_id, action_name) - - def remove_permission(self, label): - """ - Remove a permission from a queue. - - :type label: str or unicode - :param label: The unique label associated with the permission being removed. - - :rtype: bool - :return: True if successful, False otherwise. - """ - return self.connection.remove_permission(self, label) - - def read(self, visibility_timeout=None): - """ - Read a single message from the queue. - - :type visibility_timeout: int - :param visibility_timeout: The timeout for this message in seconds - - :rtype: :class:`boto.sqs.message.Message` - :return: A single message or None if queue is empty - """ - rs = self.get_messages(1, visibility_timeout) - if len(rs) == 1: - return rs[0] - else: - return None - - def write(self, message): - """ - Add a single message to the queue. - - :type message: Message - :param message: The message to be written to the queue - - :rtype: :class:`boto.sqs.message.Message` - :return: The :class:`boto.sqs.message.Message` object that was written. - """ - new_msg = self.connection.send_message(self, message.get_body_encoded()) - message.id = new_msg.id - message.md5 = new_msg.md5 - return message - - def new_message(self, body=''): - """ - Create new message of appropriate class. - - :type body: message body - :param body: The body of the newly created message (optional). - - :rtype: :class:`boto.sqs.message.Message` - :return: A new Message object - """ - m = self.message_class(self, body) - m.queue = self - return m - - # get a variable number of messages, returns a list of messages - def get_messages(self, num_messages=1, visibility_timeout=None, - attributes=None): - """ - Get a variable number of messages. - - :type num_messages: int - :param num_messages: The maximum number of messages to read from the queue. - - :type visibility_timeout: int - :param visibility_timeout: The VisibilityTimeout for the messages read. - - :type attributes: str - :param attributes: The name of additional attribute to return with response - or All if you want all attributes. The default is to - return no additional attributes. Valid values: - All - SenderId - SentTimestamp - ApproximateReceiveCount - ApproximateFirstReceiveTimestamp - - :rtype: list - :return: A list of :class:`boto.sqs.message.Message` objects. - """ - return self.connection.receive_message(self, number_messages=num_messages, - visibility_timeout=visibility_timeout, - attributes=attributes) - - def delete_message(self, message): - """ - Delete a message from the queue. - - :type message: :class:`boto.sqs.message.Message` - :param message: The :class:`boto.sqs.message.Message` object to delete. - - :rtype: bool - :return: True if successful, False otherwise - """ - return self.connection.delete_message(self, message) - - def delete(self): - """ - Delete the queue. - """ - return self.connection.delete_queue(self) - - def clear(self, page_size=10, vtimeout=10): - """Utility function to remove all messages from a queue""" - n = 0 - l = self.get_messages(page_size, vtimeout) - while l: - for m in l: - self.delete_message(m) - n += 1 - l = self.get_messages(page_size, vtimeout) - return n - - def count(self, page_size=10, vtimeout=10): - """ - Utility function to count the number of messages in a queue. - Note: This function now calls GetQueueAttributes to obtain - an 'approximate' count of the number of messages in a queue. - """ - a = self.get_attributes('ApproximateNumberOfMessages') - return int(a['ApproximateNumberOfMessages']) - - def count_slow(self, page_size=10, vtimeout=10): - """ - Deprecated. This is the old 'count' method that actually counts - the messages by reading them all. This gives an accurate count but - is very slow for queues with non-trivial number of messasges. - Instead, use get_attribute('ApproximateNumberOfMessages') to take - advantage of the new SQS capability. This is retained only for - the unit tests. - """ - n = 0 - l = self.get_messages(page_size, vtimeout) - while l: - for m in l: - n += 1 - l = self.get_messages(page_size, vtimeout) - return n - - def dump(self, file_name, page_size=10, vtimeout=10, sep='\n'): - """Utility function to dump the messages in a queue to a file - NOTE: Page size must be < 10 else SQS errors""" - fp = open(file_name, 'wb') - n = 0 - l = self.get_messages(page_size, vtimeout) - while l: - for m in l: - fp.write(m.get_body()) - if sep: - fp.write(sep) - n += 1 - l = self.get_messages(page_size, vtimeout) - fp.close() - return n - - def save_to_file(self, fp, sep='\n'): - """ - Read all messages from the queue and persist them to file-like object. - Messages are written to the file and the 'sep' string is written - in between messages. Messages are deleted from the queue after - being written to the file. - Returns the number of messages saved. - """ - n = 0 - m = self.read() - while m: - n += 1 - fp.write(m.get_body()) - if sep: - fp.write(sep) - self.delete_message(m) - m = self.read() - return n - - def save_to_filename(self, file_name, sep='\n'): - """ - Read all messages from the queue and persist them to local file. - Messages are written to the file and the 'sep' string is written - in between messages. Messages are deleted from the queue after - being written to the file. - Returns the number of messages saved. - """ - fp = open(file_name, 'wb') - n = self.save_to_file(fp, sep) - fp.close() - return n - - # for backwards compatibility - save = save_to_filename - - def save_to_s3(self, bucket): - """ - Read all messages from the queue and persist them to S3. - Messages are stored in the S3 bucket using a naming scheme of:: - - / - - Messages are deleted from the queue after being saved to S3. - Returns the number of messages saved. - """ - n = 0 - m = self.read() - while m: - n += 1 - key = bucket.new_key('%s/%s' % (self.id, m.id)) - key.set_contents_from_string(m.get_body()) - self.delete_message(m) - m = self.read() - return n - - def load_from_s3(self, bucket, prefix=None): - """ - Load messages previously saved to S3. - """ - n = 0 - if prefix: - prefix = '%s/' % prefix - else: - prefix = '%s/' % self.id[1:] - rs = bucket.list(prefix=prefix) - for key in rs: - n += 1 - m = self.new_message(key.get_contents_as_string()) - self.write(m) - return n - - def load_from_file(self, fp, sep='\n'): - """Utility function to load messages from a file-like object to a queue""" - n = 0 - body = '' - l = fp.readline() - while l: - if l == sep: - m = Message(self, body) - self.write(m) - n += 1 - print 'writing message %d' % n - body = '' - else: - body = body + l - l = fp.readline() - return n - - def load_from_filename(self, file_name, sep='\n'): - """Utility function to load messages from a local filename to a queue""" - fp = open(file_name, 'rb') - n = self.load_file_file(fp, sep) - fp.close() - return n - - # for backward compatibility - load = load_from_filename - diff --git a/backup/src/boto/sqs/regioninfo.py b/backup/src/boto/sqs/regioninfo.py deleted file mode 100644 index 66d6733..0000000 --- a/backup/src/boto/sqs/regioninfo.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -from boto.regioninfo import RegionInfo - -class SQSRegionInfo(RegionInfo): - - def __init__(self, connection=None, name=None, endpoint=None): - from boto.sqs.connection import SQSConnection - RegionInfo.__init__(self, connection, name, endpoint, - SQSConnection) diff --git a/backup/src/boto/storage_uri.py b/backup/src/boto/storage_uri.py deleted file mode 100644 index 9c051a4..0000000 --- a/backup/src/boto/storage_uri.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright 2010 Google Inc. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -import os -from boto.exception import BotoClientError -from boto.exception import InvalidUriError - - -class StorageUri(object): - """ - Base class for representing storage provider-independent bucket and - object name with a shorthand URI-like syntax. - - This is an abstract class: the constructor cannot be called (throws an - exception if you try). - """ - - connection = None - - def __init__(self): - """Uncallable constructor on abstract base StorageUri class. - """ - raise BotoClientError('Attempt to instantiate abstract StorageUri ' - 'class') - - def __repr__(self): - """Returns string representation of URI.""" - return self.uri - - def equals(self, uri): - """Returns true if two URIs are equal.""" - return self.uri == uri.uri - - def check_response(self, resp, level, uri): - if resp is None: - raise InvalidUriError('Attempt to get %s for "%s" failed. This ' - 'probably indicates the URI is invalid' % - (level, uri)) - - def connect(self, access_key_id=None, secret_access_key=None, **kwargs): - """ - Opens a connection to appropriate provider, depending on provider - portion of URI. Requires Credentials defined in boto config file (see - boto/pyami/config.py). - @type storage_uri: StorageUri - @param storage_uri: StorageUri specifying a bucket or a bucket+object - @rtype: L{AWSAuthConnection} - @return: A connection to storage service provider of the given URI. - """ - - if not self.connection: - if self.scheme == 's3': - from boto.s3.connection import S3Connection - self.connection = S3Connection(access_key_id, - secret_access_key, **kwargs) - elif self.scheme == 'gs': - from boto.gs.connection import GSConnection - self.connection = GSConnection(access_key_id, - secret_access_key, **kwargs) - elif self.scheme == 'file': - from boto.file.connection import FileConnection - self.connection = FileConnection(self) - else: - raise InvalidUriError('Unrecognized scheme "%s"' % - self.scheme) - self.connection.debug = self.debug - return self.connection - - def delete_key(self, validate=True, headers=None, version_id=None, - mfa_token=None): - if not self.object_name: - raise InvalidUriError('delete_key on object-less URI (%s)' % - self.uri) - bucket = self.get_bucket(validate, headers) - return bucket.delete_key(self.object_name, headers, version_id, - mfa_token) - - def get_all_keys(self, validate=True, headers=None): - bucket = self.get_bucket(validate, headers) - return bucket.get_all_keys(headers) - - def get_bucket(self, validate=True, headers=None): - if self.bucket_name is None: - raise InvalidUriError('get_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - bucket = conn.get_bucket(self.bucket_name, validate, headers) - self.check_response(bucket, 'bucket', self.uri) - return bucket - - def get_key(self, validate=True, headers=None, version_id=None): - if not self.object_name: - raise InvalidUriError('get_key on object-less URI (%s)' % self.uri) - bucket = self.get_bucket(validate, headers) - key = bucket.get_key(self.object_name, headers, version_id) - self.check_response(key, 'key', self.uri) - return key - - def new_key(self, validate=True, headers=None): - if not self.object_name: - raise InvalidUriError('new_key on object-less URI (%s)' % self.uri) - bucket = self.get_bucket(validate, headers) - return bucket.new_key(self.object_name) - - def get_contents_as_string(self, validate=True, headers=None, cb=None, - num_cb=10, torrent=False, version_id=None): - if not self.object_name: - raise InvalidUriError('get_contents_as_string on object-less URI ' - '(%s)' % self.uri) - key = self.get_key(validate, headers) - self.check_response(key, 'key', self.uri) - return key.get_contents_as_string(headers, cb, num_cb, torrent, - version_id) - - def acl_class(self): - if self.bucket_name is None: - raise InvalidUriError('acl_class on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - acl_class = conn.provider.acl_class - self.check_response(acl_class, 'acl_class', self.uri) - return acl_class - - def canned_acls(self): - if self.bucket_name is None: - raise InvalidUriError('canned_acls on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - canned_acls = conn.provider.canned_acls - self.check_response(canned_acls, 'canned_acls', self.uri) - return canned_acls - - -class BucketStorageUri(StorageUri): - """ - StorageUri subclass that handles bucket storage providers. - Callers should instantiate this class by calling boto.storage_uri(). - """ - - def __init__(self, scheme, bucket_name=None, object_name=None, - debug=0): - """Instantiate a BucketStorageUri from scheme,bucket,object tuple. - - @type scheme: string - @param scheme: URI scheme naming the storage provider (gs, s3, etc.) - @type bucket_name: string - @param bucket_name: bucket name - @type object_name: string - @param object_name: object name - @type debug: int - @param debug: debug level to pass in to connection (range 0..2) - - After instantiation the components are available in the following - fields: uri, scheme, bucket_name, object_name. - """ - - self.scheme = scheme - self.bucket_name = bucket_name - self.object_name = object_name - if self.bucket_name and self.object_name: - self.uri = ('%s://%s/%s' % (self.scheme, self.bucket_name, - self.object_name)) - elif self.bucket_name: - self.uri = ('%s://%s/' % (self.scheme, self.bucket_name)) - else: - self.uri = ('%s://' % self.scheme) - self.debug = debug - - def clone_replace_name(self, new_name): - """Instantiate a BucketStorageUri from the current BucketStorageUri, - but replacing the object_name. - - @type new_name: string - @param new_name: new object name - """ - if not self.bucket_name: - raise InvalidUriError('clone_replace_name() on bucket-less URI %s' % - self.uri) - return BucketStorageUri(self.scheme, self.bucket_name, new_name, - self.debug) - - def get_acl(self, validate=True, headers=None, version_id=None): - if not self.bucket_name: - raise InvalidUriError('get_acl on bucket-less URI (%s)' % self.uri) - bucket = self.get_bucket(validate, headers) - # This works for both bucket- and object- level ACLs (former passes - # key_name=None): - acl = bucket.get_acl(self.object_name, headers, version_id) - self.check_response(acl, 'acl', self.uri) - return acl - - def add_group_email_grant(self, permission, email_address, recursive=False, - validate=True, headers=None): - if self.scheme != 'gs': - raise ValueError('add_group_email_grant() not supported for %s ' - 'URIs.' % self.scheme) - if self.object_name: - if recursive: - raise ValueError('add_group_email_grant() on key-ful URI cannot ' - 'specify recursive=True') - key = self.get_key(validate, headers) - self.check_response(key, 'key', self.uri) - key.add_group_email_grant(permission, email_address, headers) - elif self.bucket_name: - bucket = self.get_bucket(validate, headers) - bucket.add_group_email_grant(permission, email_address, recursive, - headers) - else: - raise InvalidUriError('add_group_email_grant() on bucket-less URI %s' % - self.uri) - - def add_email_grant(self, permission, email_address, recursive=False, - validate=True, headers=None): - if not self.bucket_name: - raise InvalidUriError('add_email_grant on bucket-less URI (%s)' % - self.uri) - if not self.object_name: - bucket = self.get_bucket(validate, headers) - bucket.add_email_grant(permission, email_address, recursive, - headers) - else: - key = self.get_key(validate, headers) - self.check_response(key, 'key', self.uri) - key.add_email_grant(permission, email_address) - - def add_user_grant(self, permission, user_id, recursive=False, - validate=True, headers=None): - if not self.bucket_name: - raise InvalidUriError('add_user_grant on bucket-less URI (%s)' % - self.uri) - if not self.object_name: - bucket = self.get_bucket(validate, headers) - bucket.add_user_grant(permission, user_id, recursive, headers) - else: - key = self.get_key(validate, headers) - self.check_response(key, 'key', self.uri) - key.add_user_grant(permission, user_id) - - def list_grants(self, headers=None): - if not self.bucket_name: - raise InvalidUriError('list_grants on bucket-less URI (%s)' % - self.uri) - bucket = self.get_bucket(headers) - return bucket.list_grants(headers) - - def names_container(self): - """Returns True if this URI names a bucket (vs. an object). - """ - return not self.object_name - - def names_singleton(self): - """Returns True if this URI names an object (vs. a bucket). - """ - return self.object_name - - def is_file_uri(self): - return False - - def is_cloud_uri(self): - return True - - def create_bucket(self, headers=None, location='', policy=None): - if self.bucket_name is None: - raise InvalidUriError('create_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - return conn.create_bucket(self.bucket_name, headers, location, policy) - - def delete_bucket(self, headers=None): - if self.bucket_name is None: - raise InvalidUriError('delete_bucket on bucket-less URI (%s)' % - self.uri) - conn = self.connect() - return conn.delete_bucket(self.bucket_name, headers) - - def get_all_buckets(self, headers=None): - conn = self.connect() - return conn.get_all_buckets(headers) - - def get_provider(self): - conn = self.connect() - provider = conn.provider - self.check_response(provider, 'provider', self.uri) - return provider - - def set_acl(self, acl_or_str, key_name='', validate=True, headers=None, - version_id=None): - if not self.bucket_name: - raise InvalidUriError('set_acl on bucket-less URI (%s)' % - self.uri) - self.get_bucket(validate, headers).set_acl(acl_or_str, key_name, - headers, version_id) - - def set_canned_acl(self, acl_str, validate=True, headers=None, - version_id=None): - if not self.object_name: - raise InvalidUriError('set_canned_acl on object-less URI (%s)' % - self.uri) - key = self.get_key(validate, headers) - self.check_response(key, 'key', self.uri) - key.set_canned_acl(acl_str, headers, version_id) - - def set_contents_from_string(self, s, headers=None, replace=True, - cb=None, num_cb=10, policy=None, md5=None, - reduced_redundancy=False): - key = self.new_key(headers=headers) - key.set_contents_from_string(s, headers, replace, cb, num_cb, policy, - md5, reduced_redundancy) - - - -class FileStorageUri(StorageUri): - """ - StorageUri subclass that handles files in the local file system. - Callers should instantiate this class by calling boto.storage_uri(). - - See file/README about how we map StorageUri operations onto a file system. - """ - - def __init__(self, object_name, debug): - """Instantiate a FileStorageUri from a path name. - - @type object_name: string - @param object_name: object name - @type debug: boolean - @param debug: whether to enable debugging on this StorageUri - - After instantiation the components are available in the following - fields: uri, scheme, bucket_name (always blank for this "anonymous" - bucket), object_name. - """ - - self.scheme = 'file' - self.bucket_name = '' - self.object_name = object_name - self.uri = 'file://' + object_name - self.debug = debug - - def clone_replace_name(self, new_name): - """Instantiate a FileStorageUri from the current FileStorageUri, - but replacing the object_name. - - @type new_name: string - @param new_name: new object name - """ - return FileStorageUri(new_name, self.debug) - - def names_container(self): - """Returns True if this URI names a directory. - """ - return os.path.isdir(self.object_name) - - def names_singleton(self): - """Returns True if this URI names a file. - """ - return os.path.isfile(self.object_name) - - def is_file_uri(self): - return True - - def is_cloud_uri(self): - return False diff --git a/backup/src/boto/utils.py b/backup/src/boto/utils.py deleted file mode 100644 index 6bad25d..0000000 --- a/backup/src/boto/utils.py +++ /dev/null @@ -1,607 +0,0 @@ -# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ -# Copyright (c) 2010, Eucalyptus Systems, Inc. -# All rights reserved. -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -# -# Parts of this code were copied or derived from sample code supplied by AWS. -# The following notice applies to that code. -# -# This software code is made available "AS IS" without warranties of any -# kind. You may copy, display, modify and redistribute the software -# code either by itself or as incorporated into your code; provided that -# you do not remove any proprietary notices. Your use of this software -# code is at your own risk and you waive any claim against Amazon -# Digital Services, Inc. or its affiliates with respect to your use of -# this software code. (c) 2006 Amazon Digital Services, Inc. or its -# affiliates. - -""" -Some handy utility functions used by several classes. -""" - -import urllib -import urllib2 -import imp -import subprocess -import StringIO -import time -import logging.handlers -import boto -import tempfile -import smtplib -import datetime -from email.MIMEMultipart import MIMEMultipart -from email.MIMEBase import MIMEBase -from email.MIMEText import MIMEText -from email.Utils import formatdate -from email import Encoders - -try: - import hashlib - _hashfn = hashlib.sha512 -except ImportError: - import md5 - _hashfn = md5.md5 - -# List of Query String Arguments of Interest -qsa_of_interest = ['acl', 'location', 'logging', 'partNumber', 'policy', - 'requestPayment', 'torrent', 'versioning', 'versionId', - 'versions', 'website', 'uploads', 'uploadId', - 'response-content-type', 'response-content-language', - 'response-expires', 'reponse-cache-control', - 'response-content-disposition', - 'response-content-encoding'] - -# generates the aws canonical string for the given parameters -def canonical_string(method, path, headers, expires=None, - provider=None): - if not provider: - provider = boto.provider.get_default() - interesting_headers = {} - for key in headers: - lk = key.lower() - if headers[key] != None and (lk in ['content-md5', 'content-type', 'date'] or - lk.startswith(provider.header_prefix)): - interesting_headers[lk] = headers[key].strip() - - # these keys get empty strings if they don't exist - if not interesting_headers.has_key('content-type'): - interesting_headers['content-type'] = '' - if not interesting_headers.has_key('content-md5'): - interesting_headers['content-md5'] = '' - - # just in case someone used this. it's not necessary in this lib. - if interesting_headers.has_key(provider.date_header): - interesting_headers['date'] = '' - - # if you're using expires for query string auth, then it trumps date - # (and provider.date_header) - if expires: - interesting_headers['date'] = str(expires) - - sorted_header_keys = interesting_headers.keys() - sorted_header_keys.sort() - - buf = "%s\n" % method - for key in sorted_header_keys: - val = interesting_headers[key] - if key.startswith(provider.header_prefix): - buf += "%s:%s\n" % (key, val) - else: - buf += "%s\n" % val - - # don't include anything after the first ? in the resource... - # unless it is one of the QSA of interest, defined above - t = path.split('?') - buf += t[0] - - if len(t) > 1: - qsa = t[1].split('&') - qsa = [ a.split('=') for a in qsa] - qsa = [ a for a in qsa if a[0] in qsa_of_interest ] - if len(qsa) > 0: - qsa.sort(cmp=lambda x,y:cmp(x[0], y[0])) - qsa = [ '='.join(a) for a in qsa ] - buf += '?' - buf += '&'.join(qsa) - - return buf - -def merge_meta(headers, metadata, provider=None): - if not provider: - provider = boto.provider.get_default() - metadata_prefix = provider.metadata_prefix - final_headers = headers.copy() - for k in metadata.keys(): - if k.lower() in ['cache-control', 'content-md5', 'content-type', - 'content-encoding', 'content-disposition', - 'date', 'expires']: - final_headers[k] = metadata[k] - else: - final_headers[metadata_prefix + k] = metadata[k] - - return final_headers - -def get_aws_metadata(headers, provider=None): - if not provider: - provider = boto.provider.get_default() - metadata_prefix = provider.metadata_prefix - metadata = {} - for hkey in headers.keys(): - if hkey.lower().startswith(metadata_prefix): - val = urllib.unquote_plus(headers[hkey]) - metadata[hkey[len(metadata_prefix):]] = unicode(val, 'utf-8') - del headers[hkey] - return metadata - -def retry_url(url, retry_on_404=True): - for i in range(0, 10): - try: - req = urllib2.Request(url) - resp = urllib2.urlopen(req) - return resp.read() - except urllib2.HTTPError, e: - # in 2.6 you use getcode(), in 2.5 and earlier you use code - if hasattr(e, 'getcode'): - code = e.getcode() - else: - code = e.code - if code == 404 and not retry_on_404: - return '' - except: - pass - boto.log.exception('Caught exception reading instance data') - time.sleep(2**i) - boto.log.error('Unable to read instance data, giving up') - return '' - -def _get_instance_metadata(url): - d = {} - data = retry_url(url) - if data: - fields = data.split('\n') - for field in fields: - if field.endswith('/'): - d[field[0:-1]] = _get_instance_metadata(url + field) - else: - p = field.find('=') - if p > 0: - key = field[p+1:] - resource = field[0:p] + '/openssh-key' - else: - key = resource = field - val = retry_url(url + resource) - p = val.find('\n') - if p > 0: - val = val.split('\n') - d[key] = val - return d - -def get_instance_metadata(version='latest'): - """ - Returns the instance metadata as a nested Python dictionary. - Simple values (e.g. local_hostname, hostname, etc.) will be - stored as string values. Values such as ancestor-ami-ids will - be stored in the dict as a list of string values. More complex - fields such as public-keys and will be stored as nested dicts. - """ - url = 'http://169.254.169.254/%s/meta-data/' % version - return _get_instance_metadata(url) - -def get_instance_userdata(version='latest', sep=None): - url = 'http://169.254.169.254/%s/user-data' % version - user_data = retry_url(url, retry_on_404=False) - if user_data: - if sep: - l = user_data.split(sep) - user_data = {} - for nvpair in l: - t = nvpair.split('=') - user_data[t[0].strip()] = t[1].strip() - return user_data - -ISO8601 = '%Y-%m-%dT%H:%M:%SZ' - -def get_ts(ts=None): - if not ts: - ts = time.gmtime() - return time.strftime(ISO8601, ts) - -def parse_ts(ts): - return datetime.datetime.strptime(ts, ISO8601) - -def find_class(module_name, class_name=None): - if class_name: - module_name = "%s.%s" % (module_name, class_name) - modules = module_name.split('.') - c = None - - try: - for m in modules[1:]: - if c: - c = getattr(c, m) - else: - c = getattr(__import__(".".join(modules[0:-1])), m) - return c - except: - return None - -def update_dme(username, password, dme_id, ip_address): - """ - Update your Dynamic DNS record with DNSMadeEasy.com - """ - dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' - dme_url += '?username=%s&password=%s&id=%s&ip=%s' - s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) - return s.read() - -def fetch_file(uri, file=None, username=None, password=None): - """ - Fetch a file based on the URI provided. If you do not pass in a file pointer - a tempfile.NamedTemporaryFile, or None if the file could not be - retrieved is returned. - The URI can be either an HTTP url, or "s3://bucket_name/key_name" - """ - boto.log.info('Fetching %s' % uri) - if file == None: - file = tempfile.NamedTemporaryFile() - try: - if uri.startswith('s3://'): - bucket_name, key_name = uri[len('s3://'):].split('/', 1) - c = boto.connect_s3(aws_access_key_id=username, aws_secret_access_key=password) - bucket = c.get_bucket(bucket_name) - key = bucket.get_key(key_name) - key.get_contents_to_file(file) - else: - if username and password: - passman = urllib2.HTTPPasswordMgrWithDefaultRealm() - passman.add_password(None, uri, username, password) - authhandler = urllib2.HTTPBasicAuthHandler(passman) - opener = urllib2.build_opener(authhandler) - urllib2.install_opener(opener) - s = urllib2.urlopen(uri) - file.write(s.read()) - file.seek(0) - except: - raise - boto.log.exception('Problem Retrieving file: %s' % uri) - file = None - return file - -class ShellCommand(object): - - def __init__(self, command, wait=True, fail_fast=False, cwd = None): - self.exit_code = 0 - self.command = command - self.log_fp = StringIO.StringIO() - self.wait = wait - self.fail_fast = fail_fast - self.run(cwd = cwd) - - def run(self, cwd=None): - boto.log.info('running:%s' % self.command) - self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - cwd=cwd) - if(self.wait): - while self.process.poll() == None: - time.sleep(1) - t = self.process.communicate() - self.log_fp.write(t[0]) - self.log_fp.write(t[1]) - boto.log.info(self.log_fp.getvalue()) - self.exit_code = self.process.returncode - - if self.fail_fast and self.exit_code != 0: - raise Exception("Command " + self.command + " failed with status " + self.exit_code) - - return self.exit_code - - def setReadOnly(self, value): - raise AttributeError - - def getStatus(self): - return self.exit_code - - status = property(getStatus, setReadOnly, None, 'The exit code for the command') - - def getOutput(self): - return self.log_fp.getvalue() - - output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') - -class AuthSMTPHandler(logging.handlers.SMTPHandler): - """ - This class extends the SMTPHandler in the standard Python logging module - to accept a username and password on the constructor and to then use those - credentials to authenticate with the SMTP server. To use this, you could - add something like this in your boto config file: - - [handler_hand07] - class=boto.utils.AuthSMTPHandler - level=WARN - formatter=form07 - args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') - """ - - def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): - """ - Initialize the handler. - - We have extended the constructor to accept a username/password - for SMTP authentication. - """ - logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) - self.username = username - self.password = password - - def emit(self, record): - """ - Emit a record. - - Format the record and send it to the specified addressees. - It would be really nice if I could add authorization to this class - without having to resort to cut and paste inheritance but, no. - """ - try: - port = self.mailport - if not port: - port = smtplib.SMTP_PORT - smtp = smtplib.SMTP(self.mailhost, port) - smtp.login(self.username, self.password) - msg = self.format(record) - msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( - self.fromaddr, - ','.join(self.toaddrs), - self.getSubject(record), - formatdate(), msg) - smtp.sendmail(self.fromaddr, self.toaddrs, msg) - smtp.quit() - except (KeyboardInterrupt, SystemExit): - raise - except: - self.handleError(record) - -class LRUCache(dict): - """A dictionary-like object that stores only a certain number of items, and - discards its least recently used item when full. - - >>> cache = LRUCache(3) - >>> cache['A'] = 0 - >>> cache['B'] = 1 - >>> cache['C'] = 2 - >>> len(cache) - 3 - - >>> cache['A'] - 0 - - Adding new items to the cache does not increase its size. Instead, the least - recently used item is dropped: - - >>> cache['D'] = 3 - >>> len(cache) - 3 - >>> 'B' in cache - False - - Iterating over the cache returns the keys, starting with the most recently - used: - - >>> for key in cache: - ... print key - D - A - C - - This code is based on the LRUCache class from Genshi which is based on - Mighty's LRUCache from ``myghtyutils.util``, written - by Mike Bayer and released under the MIT license (Genshi uses the - BSD License). See: - - http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py - """ - - class _Item(object): - def __init__(self, key, value): - self.previous = self.next = None - self.key = key - self.value = value - def __repr__(self): - return repr(self.value) - - def __init__(self, capacity): - self._dict = dict() - self.capacity = capacity - self.head = None - self.tail = None - - def __contains__(self, key): - return key in self._dict - - def __iter__(self): - cur = self.head - while cur: - yield cur.key - cur = cur.next - - def __len__(self): - return len(self._dict) - - def __getitem__(self, key): - item = self._dict[key] - self._update_item(item) - return item.value - - def __setitem__(self, key, value): - item = self._dict.get(key) - if item is None: - item = self._Item(key, value) - self._dict[key] = item - self._insert_item(item) - else: - item.value = value - self._update_item(item) - self._manage_size() - - def __repr__(self): - return repr(self._dict) - - def _insert_item(self, item): - item.previous = None - item.next = self.head - if self.head is not None: - self.head.previous = item - else: - self.tail = item - self.head = item - self._manage_size() - - def _manage_size(self): - while len(self._dict) > self.capacity: - del self._dict[self.tail.key] - if self.tail != self.head: - self.tail = self.tail.previous - self.tail.next = None - else: - self.head = self.tail = None - - def _update_item(self, item): - if self.head == item: - return - - previous = item.previous - previous.next = item.next - if item.next is not None: - item.next.previous = previous - else: - self.tail = previous - - item.previous = None - item.next = self.head - self.head.previous = self.head = item - -class Password(object): - """ - Password object that stores itself as SHA512 hashed. - """ - def __init__(self, str=None): - """ - Load the string from an initial value, this should be the raw SHA512 hashed password - """ - self.str = str - - def set(self, value): - self.str = _hashfn(value).hexdigest() - - def __str__(self): - return str(self.str) - - def __eq__(self, other): - if other == None: - return False - return str(_hashfn(other).hexdigest()) == str(self.str) - - def __len__(self): - if self.str: - return len(self.str) - else: - return 0 - -def notify(subject, body=None, html_body=None, to_string=None, attachments=[], append_instance_id=True): - if append_instance_id: - subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject) - if not to_string: - to_string = boto.config.get_value('Notification', 'smtp_to', None) - if to_string: - try: - from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') - msg = MIMEMultipart() - msg['From'] = from_string - msg['Reply-To'] = from_string - msg['To'] = to_string - msg['Date'] = formatdate(localtime=True) - msg['Subject'] = subject - - if body: - msg.attach(MIMEText(body)) - - if html_body: - part = MIMEBase('text', 'html') - part.set_payload(html_body) - Encoders.encode_base64(part) - msg.attach(part) - - for part in attachments: - msg.attach(part) - - smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') - - # Alternate port support - if boto.config.get_value("Notification", "smtp_port"): - server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port"))) - else: - server = smtplib.SMTP(smtp_host) - - # TLS support - if boto.config.getbool("Notification", "smtp_tls"): - server.ehlo() - server.starttls() - server.ehlo() - smtp_user = boto.config.get_value('Notification', 'smtp_user', '') - smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') - if smtp_user: - server.login(smtp_user, smtp_pass) - server.sendmail(from_string, to_string, msg.as_string()) - server.quit() - except: - boto.log.exception('notify failed') - -def get_utf8_value(value): - if not isinstance(value, str) and not isinstance(value, unicode): - value = str(value) - if isinstance(value, unicode): - return value.encode('utf-8') - else: - return value - -def mklist(value): - if not isinstance(value, list): - if isinstance(value, tuple): - value = list(value) - else: - value = [value] - return value - -def pythonize_name(name, sep='_'): - s = '' - if name[0].isupper: - s = name[0].lower() - for c in name[1:]: - if c.isupper(): - s += sep + c.lower() - else: - s += c - return s - -def awsify_name(name): - return name[0:1].upper()+name[1:] diff --git a/backup/src/boto/vpc/__init__.py b/backup/src/boto/vpc/__init__.py deleted file mode 100644 index 76eea82..0000000 --- a/backup/src/boto/vpc/__init__.py +++ /dev/null @@ -1,473 +0,0 @@ -# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a connection to the EC2 service. -""" - -from boto.ec2.connection import EC2Connection -from boto.vpc.vpc import VPC -from boto.vpc.customergateway import CustomerGateway -from boto.vpc.vpngateway import VpnGateway, Attachment -from boto.vpc.dhcpoptions import DhcpOptions -from boto.vpc.subnet import Subnet -from boto.vpc.vpnconnection import VpnConnection - -class VPCConnection(EC2Connection): - - # VPC methods - - def get_all_vpcs(self, vpc_ids=None, filters=None): - """ - Retrieve information about your VPCs. You can filter results to - return information only about those VPCs that match your search - parameters. Otherwise, all VPCs associated with your account - are returned. - - :type vpc_ids: list - :param vpc_ids: A list of strings with the desired VPC ID's - - :type filters: list of tuples - :param filters: A list of tuples containing filters. Each tuple - consists of a filter key and a filter value. - Possible filter keys are: - - - *state*, the state of the VPC (pending or available) - - *cidrBlock*, CIDR block of the VPC - - *dhcpOptionsId*, the ID of a set of DHCP options - - :rtype: list - :return: A list of :class:`boto.vpc.vpc.VPC` - """ - params = {} - if vpc_ids: - self.build_list_params(params, vpc_ids, 'VpcId') - if filters: - i = 1 - for filter in filters: - params[('Filter.%d.Key' % i)] = filter[0] - params[('Filter.%d.Value.1')] = filter[1] - i += 1 - return self.get_list('DescribeVpcs', params, [('item', VPC)]) - - def create_vpc(self, cidr_block): - """ - Create a new Virtual Private Cloud. - - :type cidr_block: str - :param cidr_block: A valid CIDR block - - :rtype: The newly created VPC - :return: A :class:`boto.vpc.vpc.VPC` object - """ - params = {'CidrBlock' : cidr_block} - return self.get_object('CreateVpc', params, VPC) - - def delete_vpc(self, vpc_id): - """ - Delete a Virtual Private Cloud. - - :type vpc_id: str - :param vpc_id: The ID of the vpc to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'VpcId': vpc_id} - return self.get_status('DeleteVpc', params) - - # Customer Gateways - - def get_all_customer_gateways(self, customer_gateway_ids=None, filters=None): - """ - Retrieve information about your CustomerGateways. You can filter results to - return information only about those CustomerGateways that match your search - parameters. Otherwise, all CustomerGateways associated with your account - are returned. - - :type customer_gateway_ids: list - :param customer_gateway_ids: A list of strings with the desired CustomerGateway ID's - - :type filters: list of tuples - :param filters: A list of tuples containing filters. Each tuple - consists of a filter key and a filter value. - Possible filter keys are: - - - *state*, the state of the CustomerGateway - (pending,available,deleting,deleted) - - *type*, the type of customer gateway (ipsec.1) - - *ipAddress* the IP address of customer gateway's - internet-routable external inteface - - :rtype: list - :return: A list of :class:`boto.vpc.customergateway.CustomerGateway` - """ - params = {} - if customer_gateway_ids: - self.build_list_params(params, customer_gateway_ids, 'CustomerGatewayId') - if filters: - i = 1 - for filter in filters: - params[('Filter.%d.Key' % i)] = filter[0] - params[('Filter.%d.Value.1')] = filter[1] - i += 1 - return self.get_list('DescribeCustomerGateways', params, [('item', CustomerGateway)]) - - def create_customer_gateway(self, type, ip_address, bgp_asn): - """ - Create a new Customer Gateway - - :type type: str - :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' - - :type ip_address: str - :param ip_address: Internet-routable IP address for customer's gateway. - Must be a static address. - - :type bgp_asn: str - :param bgp_asn: Customer gateway's Border Gateway Protocol (BGP) - Autonomous System Number (ASN) - - :rtype: The newly created CustomerGateway - :return: A :class:`boto.vpc.customergateway.CustomerGateway` object - """ - params = {'Type' : type, - 'IpAddress' : ip_address, - 'BgpAsn' : bgp_asn} - return self.get_object('CreateCustomerGateway', params, CustomerGateway) - - def delete_customer_gateway(self, customer_gateway_id): - """ - Delete a Customer Gateway. - - :type customer_gateway_id: str - :param customer_gateway_id: The ID of the customer_gateway to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'CustomerGatewayId': customer_gateway_id} - return self.get_status('DeleteCustomerGateway', params) - - # VPN Gateways - - def get_all_vpn_gateways(self, vpn_gateway_ids=None, filters=None): - """ - Retrieve information about your VpnGateways. You can filter results to - return information only about those VpnGateways that match your search - parameters. Otherwise, all VpnGateways associated with your account - are returned. - - :type vpn_gateway_ids: list - :param vpn_gateway_ids: A list of strings with the desired VpnGateway ID's - - :type filters: list of tuples - :param filters: A list of tuples containing filters. Each tuple - consists of a filter key and a filter value. - Possible filter keys are: - - - *state*, the state of the VpnGateway - (pending,available,deleting,deleted) - - *type*, the type of customer gateway (ipsec.1) - - *availabilityZone*, the Availability zone the - VPN gateway is in. - - :rtype: list - :return: A list of :class:`boto.vpc.customergateway.VpnGateway` - """ - params = {} - if vpn_gateway_ids: - self.build_list_params(params, vpn_gateway_ids, 'VpnGatewayId') - if filters: - i = 1 - for filter in filters: - params[('Filter.%d.Key' % i)] = filter[0] - params[('Filter.%d.Value.1')] = filter[1] - i += 1 - return self.get_list('DescribeVpnGateways', params, [('item', VpnGateway)]) - - def create_vpn_gateway(self, type, availability_zone=None): - """ - Create a new Vpn Gateway - - :type type: str - :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' - - :type availability_zone: str - :param availability_zone: The Availability Zone where you want the VPN gateway. - - :rtype: The newly created VpnGateway - :return: A :class:`boto.vpc.vpngateway.VpnGateway` object - """ - params = {'Type' : type} - if availability_zone: - params['AvailabilityZone'] = availability_zone - return self.get_object('CreateVpnGateway', params, VpnGateway) - - def delete_vpn_gateway(self, vpn_gateway_id): - """ - Delete a Vpn Gateway. - - :type vpn_gateway_id: str - :param vpn_gateway_id: The ID of the vpn_gateway to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'VpnGatewayId': vpn_gateway_id} - return self.get_status('DeleteVpnGateway', params) - - def attach_vpn_gateway(self, vpn_gateway_id, vpc_id): - """ - Attaches a VPN gateway to a VPC. - - :type vpn_gateway_id: str - :param vpn_gateway_id: The ID of the vpn_gateway to attach - - :type vpc_id: str - :param vpc_id: The ID of the VPC you want to attach the gateway to. - - :rtype: An attachment - :return: a :class:`boto.vpc.vpngateway.Attachment` - """ - params = {'VpnGatewayId': vpn_gateway_id, - 'VpcId' : vpc_id} - return self.get_object('AttachVpnGateway', params, Attachment) - - # Subnets - - def get_all_subnets(self, subnet_ids=None, filters=None): - """ - Retrieve information about your Subnets. You can filter results to - return information only about those Subnets that match your search - parameters. Otherwise, all Subnets associated with your account - are returned. - - :type subnet_ids: list - :param subnet_ids: A list of strings with the desired Subnet ID's - - :type filters: list of tuples - :param filters: A list of tuples containing filters. Each tuple - consists of a filter key and a filter value. - Possible filter keys are: - - - *state*, the state of the Subnet - (pending,available) - - *vpdId*, the ID of teh VPC the subnet is in. - - *cidrBlock*, CIDR block of the subnet - - *availabilityZone*, the Availability Zone - the subnet is in. - - - :rtype: list - :return: A list of :class:`boto.vpc.subnet.Subnet` - """ - params = {} - if subnet_ids: - self.build_list_params(params, subnet_ids, 'SubnetId') - if filters: - i = 1 - for filter in filters: - params[('Filter.%d.Key' % i)] = filter[0] - params[('Filter.%d.Value.1' % i)] = filter[1] - i += 1 - return self.get_list('DescribeSubnets', params, [('item', Subnet)]) - - def create_subnet(self, vpc_id, cidr_block, availability_zone=None): - """ - Create a new Subnet - - :type vpc_id: str - :param vpc_id: The ID of the VPC where you want to create the subnet. - - :type cidr_block: str - :param cidr_block: The CIDR block you want the subnet to cover. - - :type availability_zone: str - :param availability_zone: The AZ you want the subnet in - - :rtype: The newly created Subnet - :return: A :class:`boto.vpc.customergateway.Subnet` object - """ - params = {'VpcId' : vpc_id, - 'CidrBlock' : cidr_block} - if availability_zone: - params['AvailabilityZone'] = availability_zone - return self.get_object('CreateSubnet', params, Subnet) - - def delete_subnet(self, subnet_id): - """ - Delete a subnet. - - :type subnet_id: str - :param subnet_id: The ID of the subnet to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'SubnetId': subnet_id} - return self.get_status('DeleteSubnet', params) - - - # DHCP Options - - def get_all_dhcp_options(self, dhcp_options_ids=None): - """ - Retrieve information about your DhcpOptions. - - :type dhcp_options_ids: list - :param dhcp_options_ids: A list of strings with the desired DhcpOption ID's - - :rtype: list - :return: A list of :class:`boto.vpc.dhcpoptions.DhcpOptions` - """ - params = {} - if dhcp_options_ids: - self.build_list_params(params, dhcp_options_ids, 'DhcpOptionsId') - return self.get_list('DescribeDhcpOptions', params, [('item', DhcpOptions)]) - - def create_dhcp_options(self, vpc_id, cidr_block, availability_zone=None): - """ - Create a new DhcpOption - - :type vpc_id: str - :param vpc_id: The ID of the VPC where you want to create the subnet. - - :type cidr_block: str - :param cidr_block: The CIDR block you want the subnet to cover. - - :type availability_zone: str - :param availability_zone: The AZ you want the subnet in - - :rtype: The newly created DhcpOption - :return: A :class:`boto.vpc.customergateway.DhcpOption` object - """ - params = {'VpcId' : vpc_id, - 'CidrBlock' : cidr_block} - if availability_zone: - params['AvailabilityZone'] = availability_zone - return self.get_object('CreateDhcpOption', params, DhcpOptions) - - def delete_dhcp_options(self, dhcp_options_id): - """ - Delete a DHCP Options - - :type dhcp_options_id: str - :param dhcp_options_id: The ID of the DHCP Options to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'DhcpOptionsId': dhcp_options_id} - return self.get_status('DeleteDhcpOptions', params) - - def associate_dhcp_options(self, dhcp_options_id, vpc_id): - """ - Associate a set of Dhcp Options with a VPC. - - :type dhcp_options_id: str - :param dhcp_options_id: The ID of the Dhcp Options - - :type vpc_id: str - :param vpc_id: The ID of the VPC. - - :rtype: bool - :return: True if successful - """ - params = {'DhcpOptionsId': dhcp_options_id, - 'VpcId' : vpc_id} - return self.get_status('AssociateDhcpOptions', params) - - # VPN Connection - - def get_all_vpn_connections(self, vpn_connection_ids=None, filters=None): - """ - Retrieve information about your VPN_CONNECTIONs. You can filter results to - return information only about those VPN_CONNECTIONs that match your search - parameters. Otherwise, all VPN_CONNECTIONs associated with your account - are returned. - - :type vpn_connection_ids: list - :param vpn_connection_ids: A list of strings with the desired VPN_CONNECTION ID's - - :type filters: list of tuples - :param filters: A list of tuples containing filters. Each tuple - consists of a filter key and a filter value. - Possible filter keys are: - - - *state*, the state of the VPN_CONNECTION - pending,available,deleting,deleted - - *type*, the type of connection, currently 'ipsec.1' - - *customerGatewayId*, the ID of the customer gateway - associated with the VPN - - *vpnGatewayId*, the ID of the VPN gateway associated - with the VPN connection - - :rtype: list - :return: A list of :class:`boto.vpn_connection.vpnconnection.VpnConnection` - """ - params = {} - if vpn_connection_ids: - self.build_list_params(params, vpn_connection_ids, 'Vpn_ConnectionId') - if filters: - i = 1 - for filter in filters: - params[('Filter.%d.Key' % i)] = filter[0] - params[('Filter.%d.Value.1')] = filter[1] - i += 1 - return self.get_list('DescribeVpnConnections', params, [('item', VpnConnection)]) - - def create_vpn_connection(self, type, customer_gateway_id, vpn_gateway_id): - """ - Create a new VPN Connection. - - :type type: str - :param type: The type of VPN Connection. Currently only 'ipsec.1' - is supported - - :type customer_gateway_id: str - :param customer_gateway_id: The ID of the customer gateway. - - :type vpn_gateway_id: str - :param vpn_gateway_id: The ID of the VPN gateway. - - :rtype: The newly created VpnConnection - :return: A :class:`boto.vpc.vpnconnection.VpnConnection` object - """ - params = {'Type' : type, - 'CustomerGatewayId' : customer_gateway_id, - 'VpnGatewayId' : vpn_gateway_id} - return self.get_object('CreateVpnConnection', params, VpnConnection) - - def delete_vpn_connection(self, vpn_connection_id): - """ - Delete a VPN Connection. - - :type vpn_connection_id: str - :param vpn_connection_id: The ID of the vpn_connection to be deleted. - - :rtype: bool - :return: True if successful - """ - params = {'VpnConnectionId': vpn_connection_id} - return self.get_status('DeleteVpnConnection', params) - - diff --git a/backup/src/boto/vpc/customergateway.py b/backup/src/boto/vpc/customergateway.py deleted file mode 100644 index 959d01f..0000000 --- a/backup/src/boto/vpc/customergateway.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a Customer Gateway -""" - -from boto.ec2.ec2object import TaggedEC2Object - -class CustomerGateway(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.type = None - self.state = None - self.ip_address = None - self.bgp_asn = None - - def __repr__(self): - return 'CustomerGateway:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'customerGatewayId': - self.id = value - elif name == 'ipAddress': - self.ip_address = value - elif name == 'type': - self.type = value - elif name == 'state': - self.state = value - elif name == 'bgpAsn': - self.bgp_asn = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/vpc/dhcpoptions.py b/backup/src/boto/vpc/dhcpoptions.py deleted file mode 100644 index 810d9cf..0000000 --- a/backup/src/boto/vpc/dhcpoptions.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a DHCP Options set -""" - -from boto.ec2.ec2object import TaggedEC2Object - -class DhcpValueSet(list): - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'value': - self.append(value) - -class DhcpConfigSet(dict): - - def startElement(self, name, attrs, connection): - if name == 'valueSet': - if not self.has_key(self._name): - self[self._name] = DhcpValueSet() - return self[self._name] - - def endElement(self, name, value, connection): - if name == 'key': - self._name = value - -class DhcpOptions(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.options = None - - def __repr__(self): - return 'DhcpOptions:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'dhcpConfigurationSet': - self.options = DhcpConfigSet() - return self.options - - def endElement(self, name, value, connection): - if name == 'dhcpOptionsId': - self.id = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/vpc/subnet.py b/backup/src/boto/vpc/subnet.py deleted file mode 100644 index 135e1a2..0000000 --- a/backup/src/boto/vpc/subnet.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a Subnet -""" - -from boto.ec2.ec2object import TaggedEC2Object - -class Subnet(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.state = None - self.cidr_block = None - self.available_ip_address_count = 0 - self.availability_zone = None - - def __repr__(self): - return 'Subnet:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'subnetId': - self.id = value - elif name == 'state': - self.state = value - elif name == 'cidrBlock': - self.cidr_block = value - elif name == 'availableIpAddressCount': - self.available_ip_address_count = int(value) - elif name == 'availabilityZone': - self.availability_zone = value - else: - setattr(self, name, value) - diff --git a/backup/src/boto/vpc/vpc.py b/backup/src/boto/vpc/vpc.py deleted file mode 100644 index 0539acd..0000000 --- a/backup/src/boto/vpc/vpc.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a Virtual Private Cloud. -""" - -from boto.ec2.ec2object import TaggedEC2Object - -class VPC(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.dhcp_options_id = None - self.state = None - self.cidr_block = None - - def __repr__(self): - return 'VPC:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'vpcId': - self.id = value - elif name == 'dhcpOptionsId': - self.dhcp_options_id = value - elif name == 'state': - self.state = value - elif name == 'cidrBlock': - self.cidr_block = value - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_vpc(self.id) - diff --git a/backup/src/boto/vpc/vpnconnection.py b/backup/src/boto/vpc/vpnconnection.py deleted file mode 100644 index 2e089e7..0000000 --- a/backup/src/boto/vpc/vpnconnection.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a VPN Connectionn -""" - -from boto.ec2.ec2object import EC2Object - -class VpnConnection(EC2Object): - - def __init__(self, connection=None): - EC2Object.__init__(self, connection) - self.id = None - self.state = None - self.customer_gateway_configuration = None - self.type = None - self.customer_gateway_id = None - self.vpn_gateway_id = None - - def __repr__(self): - return 'VpnConnection:%s' % self.id - - def endElement(self, name, value, connection): - if name == 'vpnConnectionId': - self.id = value - elif name == 'state': - self.state = value - elif name == 'CustomerGatewayConfiguration': - self.customer_gateway_configuration = value - elif name == 'type': - self.type = value - elif name == 'customerGatewayId': - self.customer_gateway_id = value - elif name == 'vpnGatewayId': - self.vpn_gateway_id = value - else: - setattr(self, name, value) - - def delete(self): - return self.connection.delete_vpn_connection(self.id) - diff --git a/backup/src/boto/vpc/vpngateway.py b/backup/src/boto/vpc/vpngateway.py deleted file mode 100644 index 83b912e..0000000 --- a/backup/src/boto/vpc/vpngateway.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the -# "Software"), to deal in the Software without restriction, including -# without limitation the rights to use, copy, modify, merge, publish, dis- -# tribute, sublicense, and/or sell copies of the Software, and to permit -# persons to whom the Software is furnished to do so, subject to the fol- -# lowing conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- -# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. - -""" -Represents a Vpn Gateway -""" - -from boto.ec2.ec2object import TaggedEC2Object - -class Attachment(object): - - def __init__(self, connection=None): - self.vpc_id = None - self.state = None - - def startElement(self, name, attrs, connection): - pass - - def endElement(self, name, value, connection): - if name == 'vpcId': - self.vpc_id = value - elif name == 'state': - self.state = value - else: - setattr(self, name, value) - -class VpnGateway(TaggedEC2Object): - - def __init__(self, connection=None): - TaggedEC2Object.__init__(self, connection) - self.id = None - self.type = None - self.state = None - self.availability_zone = None - self.attachments = [] - - def __repr__(self): - return 'VpnGateway:%s' % self.id - - def startElement(self, name, attrs, connection): - retval = TaggedEC2Object.startElement(self, name, attrs, connection) - if retval is not None: - return retval - if name == 'item': - att = Attachment() - self.attachments.append(att) - return att - - def endElement(self, name, value, connection): - if name == 'vpnGatewayId': - self.id = value - elif name == 'type': - self.type = value - elif name == 'state': - self.state = value - elif name == 'availabilityZone': - self.availability_zone = value - elif name == 'attachments': - pass - else: - setattr(self, name, value) - - def attach(self, vpc_id): - return self.connection.attach_vpn_gateway(self.id, vpc_id) -