diff --git a/backup/.project b/backup/.project new file mode 100644 index 0000000..e5f3418 --- /dev/null +++ b/backup/.project @@ -0,0 +1,17 @@ + + + backup + + + + + + org.python.pydev.PyDevBuilder + + + + + + org.python.pydev.pythonNature + + diff --git a/backup/.pydevproject b/backup/.pydevproject new file mode 100644 index 0000000..3a6f392 --- /dev/null +++ b/backup/.pydevproject @@ -0,0 +1,10 @@ + + + + +Default +python 2.7 + +/backup/src + + diff --git a/backup/src/boto/__init__.py b/backup/src/boto/__init__.py new file mode 100644 index 0000000..d11b578 --- /dev/null +++ b/backup/src/boto/__init__.py @@ -0,0 +1,542 @@ +# Copyright (c) 2006-2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto +from boto.pyami.config import Config, BotoConfigLocations +from boto.storage_uri import BucketStorageUri, FileStorageUri +import boto.plugin +import os, re, sys +import logging +import logging.config +from boto.exception import InvalidUriError + +__version__ = '2.0b4' +Version = __version__ # for backware compatibility + +UserAgent = 'Boto/%s (%s)' % (__version__, sys.platform) +config = Config() + +def init_logging(): + for file in BotoConfigLocations: + try: + logging.config.fileConfig(os.path.expanduser(file)) + except: + pass + +class NullHandler(logging.Handler): + def emit(self, record): + pass + +log = logging.getLogger('boto') +log.addHandler(NullHandler()) +init_logging() + +# convenience function to set logging to a particular file +def set_file_logger(name, filepath, level=logging.INFO, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.FileHandler(filepath) + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def set_stream_logger(name, level=logging.DEBUG, format_string=None): + global log + if not format_string: + format_string = "%(asctime)s %(name)s [%(levelname)s]:%(message)s" + logger = logging.getLogger(name) + logger.setLevel(level) + fh = logging.StreamHandler() + fh.setLevel(level) + formatter = logging.Formatter(format_string) + fh.setFormatter(formatter) + logger.addHandler(fh) + log = logger + +def connect_sqs(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sqs.connection.SQSConnection` + :return: A connection to Amazon's SQS + """ + from boto.sqs.connection import SQSConnection + return SQSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_s3(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.s3.connection.S3Connection` + :return: A connection to Amazon's S3 + """ + from boto.s3.connection import S3Connection + return S3Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_gs(gs_access_key_id=None, gs_secret_access_key=None, **kwargs): + """ + @type gs_access_key_id: string + @param gs_access_key_id: Your Google Storage Access Key ID + + @type gs_secret_access_key: string + @param gs_secret_access_key: Your Google Storage Secret Access Key + + @rtype: L{GSConnection} + @return: A connection to Google's Storage service + """ + from boto.gs.connection import GSConnection + return GSConnection(gs_access_key_id, gs_secret_access_key, **kwargs) + +def connect_ec2(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: A connection to Amazon's EC2 + """ + from boto.ec2.connection import EC2Connection + return EC2Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_elb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.elb.ELBConnection` + :return: A connection to Amazon's Load Balancing Service + """ + from boto.ec2.elb import ELBConnection + return ELBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_autoscale(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.autoscale.AutoScaleConnection` + :return: A connection to Amazon's Auto Scaling Service + """ + from boto.ec2.autoscale import AutoScaleConnection + return AutoScaleConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudwatch(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.cloudwatch.CloudWatchConnection` + :return: A connection to Amazon's EC2 Monitoring service + """ + from boto.ec2.cloudwatch import CloudWatchConnection + return CloudWatchConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_sdb(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sdb.connection.SDBConnection` + :return: A connection to Amazon's SDB + """ + from boto.sdb.connection import SDBConnection + return SDBConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_fps(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.fps.connection import FPSConnection + return FPSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_mturk(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.mturk.connection.MTurkConnection` + :return: A connection to MTurk + """ + from boto.mturk.connection import MTurkConnection + return MTurkConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_cloudfront(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.fps.connection.FPSConnection` + :return: A connection to FPS + """ + from boto.cloudfront import CloudFrontConnection + return CloudFrontConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_vpc(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.vpc.VPCConnection` + :return: A connection to VPC + """ + from boto.vpc import VPCConnection + return VPCConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_rds(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.rds.RDSConnection` + :return: A connection to RDS + """ + from boto.rds import RDSConnection + return RDSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_emr(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.emr.EmrConnection` + :return: A connection to Elastic mapreduce + """ + from boto.emr import EmrConnection + return EmrConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_sns(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.sns.SNSConnection` + :return: A connection to Amazon's SNS + """ + from boto.sns import SNSConnection + return SNSConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + + +def connect_iam(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.iam.IAMConnection` + :return: A connection to Amazon's IAM + """ + from boto.iam import IAMConnection + return IAMConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_route53(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.dns.Route53Connection` + :return: A connection to Amazon's Route53 DNS Service + """ + from boto.route53 import Route53Connection + return Route53Connection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_euca(host, aws_access_key_id=None, aws_secret_access_key=None, + port=8773, path='/services/Eucalyptus', is_secure=False, + **kwargs): + """ + Connect to a Eucalyptus service. + + :type host: string + :param host: the host name or ip address of the Eucalyptus server + + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ec2.connection.EC2Connection` + :return: A connection to Eucalyptus server + """ + from boto.ec2 import EC2Connection + from boto.ec2.regioninfo import RegionInfo + + reg = RegionInfo(name='eucalyptus', endpoint=host) + return EC2Connection(aws_access_key_id, aws_secret_access_key, + region=reg, port=port, path=path, + is_secure=is_secure, **kwargs) + +def connect_walrus(host, aws_access_key_id=None, aws_secret_access_key=None, + port=8773, path='/services/Walrus', is_secure=False, + **kwargs): + """ + Connect to a Walrus service. + + :type host: string + :param host: the host name or ip address of the Walrus server + + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.s3.connection.S3Connection` + :return: A connection to Walrus + """ + from boto.s3.connection import S3Connection + from boto.s3.connection import OrdinaryCallingFormat + + return S3Connection(aws_access_key_id, aws_secret_access_key, + host=host, port=port, path=path, + calling_format=OrdinaryCallingFormat(), + is_secure=is_secure, **kwargs) + +def connect_ses(aws_access_key_id=None, aws_secret_access_key=None, **kwargs): + """ + :type aws_access_key_id: string + :param aws_access_key_id: Your AWS Access Key ID + + :type aws_secret_access_key: string + :param aws_secret_access_key: Your AWS Secret Access Key + + :rtype: :class:`boto.ses.SESConnection` + :return: A connection to Amazon's SES + """ + from boto.ses import SESConnection + return SESConnection(aws_access_key_id, aws_secret_access_key, **kwargs) + +def connect_ia(ia_access_key_id=None, ia_secret_access_key=None, + is_secure=False, **kwargs): + """ + Connect to the Internet Archive via their S3-like API. + + :type ia_access_key_id: string + :param ia_access_key_id: Your IA Access Key ID. This will also look in your + boto config file for an entry in the Credentials + section called "ia_access_key_id" + + :type ia_secret_access_key: string + :param ia_secret_access_key: Your IA Secret Access Key. This will also look in your + boto config file for an entry in the Credentials + section called "ia_secret_access_key" + + :rtype: :class:`boto.s3.connection.S3Connection` + :return: A connection to the Internet Archive + """ + from boto.s3.connection import S3Connection + from boto.s3.connection import OrdinaryCallingFormat + + access_key = config.get('Credentials', 'ia_access_key_id', + ia_access_key_id) + secret_key = config.get('Credentials', 'ia_secret_access_key', + ia_secret_access_key) + + return S3Connection(access_key, secret_key, + host='s3.us.archive.org', + calling_format=OrdinaryCallingFormat(), + is_secure=is_secure, **kwargs) + +def check_extensions(module_name, module_path): + """ + This function checks for extensions to boto modules. It should be called in the + __init__.py file of all boto modules. See: + http://code.google.com/p/boto/wiki/ExtendModules + + for details. + """ + option_name = '%s_extend' % module_name + version = config.get('Boto', option_name, None) + if version: + dirname = module_path[0] + path = os.path.join(dirname, version) + if os.path.isdir(path): + log.info('extending module %s with: %s' % (module_name, path)) + module_path.insert(0, path) + +_aws_cache = {} + +def _get_aws_conn(service): + global _aws_cache + conn = _aws_cache.get(service) + if not conn: + meth = getattr(sys.modules[__name__], 'connect_' + service) + conn = meth() + _aws_cache[service] = conn + return conn + +def lookup(service, name): + global _aws_cache + conn = _get_aws_conn(service) + obj = _aws_cache.get('.'.join((service, name)), None) + if not obj: + obj = conn.lookup(name) + _aws_cache['.'.join((service, name))] = obj + return obj + +def storage_uri(uri_str, default_scheme='file', debug=0, validate=True, + bucket_storage_uri_class=BucketStorageUri): + """ + Instantiate a StorageUri from a URI string. + + :type uri_str: string + :param uri_str: URI naming bucket + optional object. + :type default_scheme: string + :param default_scheme: default scheme for scheme-less URIs. + :type debug: int + :param debug: debug level to pass in to boto connection (range 0..2). + :type validate: bool + :param validate: whether to check for bucket name validity. + :type bucket_storage_uri_class: BucketStorageUri interface. + :param bucket_storage_uri_class: Allows mocking for unit tests. + + We allow validate to be disabled to allow caller + to implement bucket-level wildcarding (outside the boto library; + see gsutil). + + :rtype: :class:`boto.StorageUri` subclass + :return: StorageUri subclass for given URI. + + ``uri_str`` must be one of the following formats: + + * gs://bucket/name + * s3://bucket/name + * gs://bucket + * s3://bucket + * filename + + The last example uses the default scheme ('file', unless overridden) + """ + + # Manually parse URI components instead of using urlparse.urlparse because + # what we're calling URIs don't really fit the standard syntax for URIs + # (the latter includes an optional host/net location part). + end_scheme_idx = uri_str.find('://') + if end_scheme_idx == -1: + # Check for common error: user specifies gs:bucket instead + # of gs://bucket. Some URI parsers allow this, but it can cause + # confusion for callers, so we don't. + if uri_str.find(':') != -1: + raise InvalidUriError('"%s" contains ":" instead of "://"' % uri_str) + scheme = default_scheme.lower() + path = uri_str + else: + scheme = uri_str[0:end_scheme_idx].lower() + path = uri_str[end_scheme_idx + 3:] + + if scheme not in ['file', 's3', 'gs']: + raise InvalidUriError('Unrecognized scheme "%s"' % scheme) + if scheme == 'file': + # For file URIs we have no bucket name, and use the complete path + # (minus 'file://') as the object name. + return FileStorageUri(path, debug) + else: + path_parts = path.split('/', 1) + bucket_name = path_parts[0] + if (validate and bucket_name and + # Disallow buckets violating charset or not [3..255] chars total. + (not re.match('^[a-z0-9][a-z0-9\._-]{1,253}[a-z0-9]$', bucket_name) + # Disallow buckets with individual DNS labels longer than 63. + or re.search('[-_a-z0-9]{64}', bucket_name))): + raise InvalidUriError('Invalid bucket name in URI "%s"' % uri_str) + # If enabled, ensure the bucket name is valid, to avoid possibly + # confusing other parts of the code. (For example if we didn't + # catch bucket names containing ':', when a user tried to connect to + # the server with that name they might get a confusing error about + # non-integer port numbers.) + object_name = '' + if len(path_parts) > 1: + object_name = path_parts[1] + return bucket_storage_uri_class(scheme, bucket_name, object_name, debug) + +def storage_uri_for_key(key): + """Returns a StorageUri for the given key. + + :type key: :class:`boto.s3.key.Key` or subclass + :param key: URI naming bucket + optional object. + """ + if not isinstance(key, boto.s3.key.Key): + raise InvalidUriError('Requested key (%s) is not a subclass of ' + 'boto.s3.key.Key' % str(type(key))) + prov_name = key.bucket.connection.provider.get_provider_name() + uri_str = '%s://%s/%s' % (prov_name, key.bucket.name, key.name) + return storage_uri(uri_str) + +boto.plugin.load_plugins(config) diff --git a/backup/src/boto/auth.py b/backup/src/boto/auth.py new file mode 100644 index 0000000..6c6c1f2 --- /dev/null +++ b/backup/src/boto/auth.py @@ -0,0 +1,319 @@ +# Copyright 2010 Google Inc. +# Copyright (c) 2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2011, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +""" +Handles authentication required to AWS and GS +""" + +import base64 +import boto +import boto.auth_handler +import boto.exception +import boto.plugin +import boto.utils +import hmac +import sys +import time +import urllib + +from boto.auth_handler import AuthHandler +from boto.exception import BotoClientError +# +# the following is necessary because of the incompatibilities +# between Python 2.4, 2.5, and 2.6 as well as the fact that some +# people running 2.4 have installed hashlib as a separate module +# this fix was provided by boto user mccormix. +# see: http://code.google.com/p/boto/issues/detail?id=172 +# for more details. +# +try: + from hashlib import sha1 as sha + from hashlib import sha256 as sha256 + + if sys.version[:3] == "2.4": + # we are using an hmac that expects a .new() method. + class Faker: + def __init__(self, which): + self.which = which + self.digest_size = self.which().digest_size + + def new(self, *args, **kwargs): + return self.which(*args, **kwargs) + + sha = Faker(sha) + sha256 = Faker(sha256) + +except ImportError: + import sha + sha256 = None + +class HmacKeys(object): + """Key based Auth handler helper.""" + + def __init__(self, host, config, provider): + if provider.access_key is None or provider.secret_key is None: + raise boto.auth_handler.NotReadyToAuthenticate() + self._provider = provider + self._hmac = hmac.new(self._provider.secret_key, digestmod=sha) + if sha256: + self._hmac_256 = hmac.new(self._provider.secret_key, digestmod=sha256) + else: + self._hmac_256 = None + + def algorithm(self): + if self._hmac_256: + return 'HmacSHA256' + else: + return 'HmacSHA1' + + def sign_string(self, string_to_sign): + boto.log.debug('Canonical: %s' % string_to_sign) + if self._hmac_256: + hmac = self._hmac_256.copy() + else: + hmac = self._hmac.copy() + hmac.update(string_to_sign) + return base64.encodestring(hmac.digest()).strip() + +class HmacAuthV1Handler(AuthHandler, HmacKeys): + """ Implements the HMAC request signing used by S3 and GS.""" + + capability = ['hmac-v1', 's3'] + + def __init__(self, host, config, provider): + AuthHandler.__init__(self, host, config, provider) + HmacKeys.__init__(self, host, config, provider) + self._hmac_256 = None + + def add_auth(self, http_request, **kwargs): + headers = http_request.headers + method = http_request.method + auth_path = http_request.auth_path + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + c_string = boto.utils.canonical_string(method, auth_path, headers, + None, self._provider) + b64_hmac = self.sign_string(c_string) + auth_hdr = self._provider.auth_header + headers['Authorization'] = ("%s %s:%s" % + (auth_hdr, + self._provider.access_key, b64_hmac)) + +class HmacAuthV2Handler(AuthHandler, HmacKeys): + """ + Implements the simplified HMAC authorization used by CloudFront. + """ + capability = ['hmac-v2', 'cloudfront'] + + def __init__(self, host, config, provider): + AuthHandler.__init__(self, host, config, provider) + HmacKeys.__init__(self, host, config, provider) + self._hmac_256 = None + + def add_auth(self, http_request, **kwargs): + headers = http_request.headers + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + b64_hmac = self.sign_string(headers['Date']) + auth_hdr = self._provider.auth_header + headers['Authorization'] = ("%s %s:%s" % + (auth_hdr, + self._provider.access_key, b64_hmac)) + +class HmacAuthV3Handler(AuthHandler, HmacKeys): + """Implements the new Version 3 HMAC authorization used by Route53.""" + + capability = ['hmac-v3', 'route53', 'ses'] + + def __init__(self, host, config, provider): + AuthHandler.__init__(self, host, config, provider) + HmacKeys.__init__(self, host, config, provider) + + def add_auth(self, http_request, **kwargs): + headers = http_request.headers + if not headers.has_key('Date'): + headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", + time.gmtime()) + + b64_hmac = self.sign_string(headers['Date']) + s = "AWS3-HTTPS AWSAccessKeyId=%s," % self._provider.access_key + s += "Algorithm=%s,Signature=%s" % (self.algorithm(), b64_hmac) + headers['X-Amzn-Authorization'] = s + +class QuerySignatureHelper(HmacKeys): + """Helper for Query signature based Auth handler. + + Concrete sub class need to implement _calc_sigature method. + """ + + def add_auth(self, http_request, **kwargs): + headers = http_request.headers + params = http_request.params + params['AWSAccessKeyId'] = self._provider.access_key + params['SignatureVersion'] = self.SignatureVersion + params['Timestamp'] = boto.utils.get_ts() + qs, signature = self._calc_signature( + http_request.params, http_request.method, + http_request.path, http_request.host) + boto.log.debug('query_string: %s Signature: %s' % (qs, signature)) + if http_request.method == 'POST': + headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8' + http_request.body = qs + '&Signature=' + urllib.quote(signature) + else: + http_request.body = '' + http_request.path = (http_request.path + '?' + qs + '&Signature=' + urllib.quote(signature)) + # Now that query params are part of the path, clear the 'params' field + # in request. + http_request.params = {} + +class QuerySignatureV0AuthHandler(QuerySignatureHelper, AuthHandler): + """Class SQS query signature based Auth handler.""" + + SignatureVersion = 0 + capability = ['sign-v0'] + + def _calc_signature(self, params, *args): + boto.log.debug('using _calc_signature_0') + hmac = self._hmac.copy() + s = params['Action'] + params['Timestamp'] + hmac.update(s) + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + val = bot.utils.get_utf8_value(params[key]) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + +class QuerySignatureV1AuthHandler(QuerySignatureHelper, AuthHandler): + """ + Provides Query Signature V1 Authentication. + """ + + SignatureVersion = 1 + capability = ['sign-v1', 'mturk'] + + def _calc_signature(self, params, *args): + boto.log.debug('using _calc_signature_1') + hmac = self._hmac.copy() + keys = params.keys() + keys.sort(cmp = lambda x, y: cmp(x.lower(), y.lower())) + pairs = [] + for key in keys: + hmac.update(key) + val = boto.utils.get_utf8_value(params[key]) + hmac.update(val) + pairs.append(key + '=' + urllib.quote(val)) + qs = '&'.join(pairs) + return (qs, base64.b64encode(hmac.digest())) + +class QuerySignatureV2AuthHandler(QuerySignatureHelper, AuthHandler): + """Provides Query Signature V2 Authentication.""" + + SignatureVersion = 2 + capability = ['sign-v2', 'ec2', 'ec2', 'emr', 'fps', 'ecs', + 'sdb', 'iam', 'rds', 'sns', 'sqs'] + + def _calc_signature(self, params, verb, path, server_name): + boto.log.debug('using _calc_signature_2') + string_to_sign = '%s\n%s\n%s\n' % (verb, server_name.lower(), path) + if self._hmac_256: + hmac = self._hmac_256.copy() + params['SignatureMethod'] = 'HmacSHA256' + else: + hmac = self._hmac.copy() + params['SignatureMethod'] = 'HmacSHA1' + keys = params.keys() + keys.sort() + pairs = [] + for key in keys: + val = boto.utils.get_utf8_value(params[key]) + pairs.append(urllib.quote(key, safe='') + '=' + + urllib.quote(val, safe='-_~')) + qs = '&'.join(pairs) + boto.log.debug('query string: %s' % qs) + string_to_sign += qs + boto.log.debug('string_to_sign: %s' % string_to_sign) + hmac.update(string_to_sign) + b64 = base64.b64encode(hmac.digest()) + boto.log.debug('len(b64)=%d' % len(b64)) + boto.log.debug('base64 encoded digest: %s' % b64) + return (qs, b64) + + +def get_auth_handler(host, config, provider, requested_capability=None): + """Finds an AuthHandler that is ready to authenticate. + + Lists through all the registered AuthHandlers to find one that is willing + to handle for the requested capabilities, config and provider. + + :type host: string + :param host: The name of the host + + :type config: + :param config: + + :type provider: + :param provider: + + Returns: + An implementation of AuthHandler. + + Raises: + boto.exception.NoAuthHandlerFound: + boto.exception.TooManyAuthHandlerReadyToAuthenticate: + """ + ready_handlers = [] + auth_handlers = boto.plugin.get_plugin(AuthHandler, requested_capability) + total_handlers = len(auth_handlers) + for handler in auth_handlers: + try: + ready_handlers.append(handler(host, config, provider)) + except boto.auth_handler.NotReadyToAuthenticate: + pass + + if not ready_handlers: + checked_handlers = auth_handlers + names = [handler.__name__ for handler in checked_handlers] + raise boto.exception.NoAuthHandlerFound( + 'No handler was ready to authenticate. %d handlers were checked.' + ' %s ' % (len(names), str(names))) + + if len(ready_handlers) > 1: + # NOTE: Even though it would be nice to accept more than one handler + # by using one of the many ready handlers, we are never sure that each + # of them are referring to the same storage account. Since we cannot + # easily guarantee that, it is always safe to fail, rather than operate + # on the wrong account. + names = [handler.__class__.__name__ for handler in ready_handlers] + raise boto.exception.TooManyAuthHandlerReadyToAuthenticate( + '%d AuthHandlers ready to authenticate, ' + 'only 1 expected: %s' % (len(names), str(names))) + + return ready_handlers[0] diff --git a/backup/src/boto/auth_handler.py b/backup/src/boto/auth_handler.py new file mode 100644 index 0000000..ab2d317 --- /dev/null +++ b/backup/src/boto/auth_handler.py @@ -0,0 +1,58 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Defines an interface which all Auth handlers need to implement. +""" + +from plugin import Plugin + +class NotReadyToAuthenticate(Exception): + pass + +class AuthHandler(Plugin): + + capability = [] + + def __init__(self, host, config, provider): + """Constructs the handlers. + :type host: string + :param host: The host to which the request is being sent. + + :type config: boto.pyami.Config + :param config: Boto configuration. + + :type provider: boto.provider.Provider + :param provider: Provider details. + + Raises: + NotReadyToAuthenticate: if this handler is not willing to + authenticate for the given provider and config. + """ + pass + + def add_auth(self, http_request): + """Invoked to add authentication details to request. + + :type http_request: boto.connection.HTTPRequest + :param http_request: HTTP request that needs to be authenticated. + """ + pass diff --git a/backup/src/boto/cloudfront/__init__.py b/backup/src/boto/cloudfront/__init__.py new file mode 100644 index 0000000..bd02b00 --- /dev/null +++ b/backup/src/boto/cloudfront/__init__.py @@ -0,0 +1,248 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import time +import boto +from boto.connection import AWSAuthConnection +from boto import handler +from boto.cloudfront.distribution import Distribution, DistributionSummary, DistributionConfig +from boto.cloudfront.distribution import StreamingDistribution, StreamingDistributionSummary, StreamingDistributionConfig +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.identity import OriginAccessIdentitySummary +from boto.cloudfront.identity import OriginAccessIdentityConfig +from boto.cloudfront.invalidation import InvalidationBatch +from boto.resultset import ResultSet +from boto.cloudfront.exception import CloudFrontServerError + +class CloudFrontConnection(AWSAuthConnection): + + DefaultHost = 'cloudfront.amazonaws.com' + Version = '2010-11-01' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + True, port, proxy, proxy_port, debug=debug) + + def get_etag(self, response): + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + return response_headers[key] + return None + + def _required_auth_capability(self): + return ['cloudfront'] + + # Generics + + def _get_all_objects(self, resource, tags): + if not tags: + tags=[('DistributionSummary', DistributionSummary)] + response = self.make_request('GET', '/%s/%s' % (self.Version, resource)) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + rs = ResultSet(tags) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def _get_info(self, id, resource, dist_class): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = dist_class(connection=self) + response_headers = response.msg + for key in response_headers.keys(): + if key.lower() == 'etag': + d.etag = response_headers[key] + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _get_config(self, id, resource, config_class): + uri = '/%s/%s/%s/config' % (self.Version, resource, id) + response = self.make_request('GET', uri) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise CloudFrontServerError(response.status, response.reason, body) + d = config_class(connection=self) + d.etag = self.get_etag(response) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + return d + + def _set_config(self, distribution_id, etag, config): + if isinstance(config, StreamingDistributionConfig): + resource = 'streaming-distribution' + else: + resource = 'distribution' + uri = '/%s/%s/%s/config' % (self.Version, resource, distribution_id) + headers = {'If-Match' : etag, 'Content-Type' : 'text/xml'} + response = self.make_request('PUT', uri, headers, config.to_xml()) + body = response.read() + boto.log.debug(body) + if response.status != 200: + raise CloudFrontServerError(response.status, response.reason, body) + return self.get_etag(response) + + def _create_object(self, config, resource, dist_class): + response = self.make_request('POST', '/%s/%s' % (self.Version, resource), + {'Content-Type' : 'text/xml'}, data=config.to_xml()) + body = response.read() + boto.log.debug(body) + if response.status == 201: + d = dist_class(connection=self) + h = handler.XmlHandler(d, self) + xml.sax.parseString(body, h) + d.etag = self.get_etag(response) + return d + else: + raise CloudFrontServerError(response.status, response.reason, body) + + def _delete_object(self, id, etag, resource): + uri = '/%s/%s/%s' % (self.Version, resource, id) + response = self.make_request('DELETE', uri, {'If-Match' : etag}) + body = response.read() + boto.log.debug(body) + if response.status != 204: + raise CloudFrontServerError(response.status, response.reason, body) + + # Distributions + + def get_all_distributions(self): + tags=[('DistributionSummary', DistributionSummary)] + return self._get_all_objects('distribution', tags) + + def get_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'distribution', Distribution) + + def get_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'distribution', + DistributionConfig) + + def set_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_distribution(self, origin, enabled, caller_reference='', + cnames=None, comment=''): + config = DistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'distribution', Distribution) + + def delete_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'distribution') + + # Streaming Distributions + + def get_all_streaming_distributions(self): + tags=[('StreamingDistributionSummary', StreamingDistributionSummary)] + return self._get_all_objects('streaming-distribution', tags) + + def get_streaming_distribution_info(self, distribution_id): + return self._get_info(distribution_id, 'streaming-distribution', + StreamingDistribution) + + def get_streaming_distribution_config(self, distribution_id): + return self._get_config(distribution_id, 'streaming-distribution', + StreamingDistributionConfig) + + def set_streaming_distribution_config(self, distribution_id, etag, config): + return self._set_config(distribution_id, etag, config) + + def create_streaming_distribution(self, origin, enabled, + caller_reference='', + cnames=None, comment=''): + config = StreamingDistributionConfig(origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment) + return self._create_object(config, 'streaming-distribution', + StreamingDistribution) + + def delete_streaming_distribution(self, distribution_id, etag): + return self._delete_object(distribution_id, etag, 'streaming-distribution') + + # Origin Access Identity + + def get_all_origin_access_identity(self): + tags=[('CloudFrontOriginAccessIdentitySummary', + OriginAccessIdentitySummary)] + return self._get_all_objects('origin-access-identity/cloudfront', tags) + + def get_origin_access_identity_info(self, access_id): + return self._get_info(access_id, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def get_origin_access_identity_config(self, access_id): + return self._get_config(access_id, + 'origin-access-identity/cloudfront', + OriginAccessIdentityConfig) + + def set_origin_access_identity_config(self, access_id, + etag, config): + return self._set_config(access_id, etag, config) + + def create_origin_access_identity(self, caller_reference='', comment=''): + config = OriginAccessIdentityConfig(caller_reference=caller_reference, + comment=comment) + return self._create_object(config, 'origin-access-identity/cloudfront', + OriginAccessIdentity) + + def delete_origin_access_identity(self, access_id, etag): + return self._delete_object(access_id, etag, + 'origin-access-identity/cloudfront') + + # Object Invalidation + + def create_invalidation_request(self, distribution_id, paths, + caller_reference=None): + """Creates a new invalidation request + :see: http://goo.gl/8vECq + """ + # We allow you to pass in either an array or + # an InvalidationBatch object + if not isinstance(paths, InvalidationBatch): + paths = InvalidationBatch(paths) + paths.connection = self + uri = '/%s/distribution/%s/invalidation' % (self.Version, + distribution_id) + response = self.make_request('POST', uri, + {'Content-Type' : 'text/xml'}, + data=paths.to_xml()) + body = response.read() + if response.status == 201: + h = handler.XmlHandler(paths, self) + xml.sax.parseString(body, h) + return paths + else: + raise CloudFrontServerError(response.status, response.reason, body) + diff --git a/backup/src/boto/cloudfront/distribution.py b/backup/src/boto/cloudfront/distribution.py new file mode 100644 index 0000000..ed245cb --- /dev/null +++ b/backup/src/boto/cloudfront/distribution.py @@ -0,0 +1,540 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid +from boto.cloudfront.identity import OriginAccessIdentity +from boto.cloudfront.object import Object, StreamingObject +from boto.cloudfront.signers import ActiveTrustedSigners, TrustedSigners +from boto.cloudfront.logging import LoggingInfo +from boto.cloudfront.origin import S3Origin, CustomOrigin +from boto.s3.acl import ACL + +class DistributionConfig: + + def __init__(self, connection=None, origin=None, enabled=False, + caller_reference='', cnames=None, comment='', + trusted_signers=None, default_root_object=None, + logging=None): + """ + :param origin: Origin information to associate with the + distribution. If your distribution will use + an Amazon S3 origin, then this should be an + S3Origin object. If your distribution will use + a custom origin (non Amazon S3), then this + should be a CustomOrigin object. + :type origin: :class:`boto.cloudfront.origin.S3Origin` or + :class:`boto.cloudfront.origin.CustomOrigin` + + :param enabled: Whether the distribution is enabled to accept + end user requests for content. + :type enabled: bool + + :param caller_reference: A unique number that ensures the + request can't be replayed. If no + caller_reference is provided, boto + will generate a type 4 UUID for use + as the caller reference. + :type enabled: str + + :param cnames: A CNAME alias you want to associate with this + distribution. You can have up to 10 CNAME aliases + per distribution. + :type enabled: array of str + + :param comment: Any comments you want to include about the + distribution. + :type comment: str + + :param trusted_signers: Specifies any AWS accounts you want to + permit to create signed URLs for private + content. If you want the distribution to + use signed URLs, this should contain a + TrustedSigners object; if you want the + distribution to use basic URLs, leave + this None. + :type trusted_signers: :class`boto.cloudfront.signers.TrustedSigners` + + :param default_root_object: Designates a default root object. + Only include a DefaultRootObject value + if you are going to assign a default + root object for the distribution. + :type comment: str + + :param logging: Controls whether access logs are written for the + distribution. If you want to turn on access logs, + this should contain a LoggingInfo object; otherwise + it should contain None. + :type logging: :class`boto.cloudfront.logging.LoggingInfo` + + """ + self.connection = connection + self.origin = origin + self.enabled = enabled + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.cnames = [] + if cnames: + self.cnames = cnames + self.comment = comment + self.trusted_signers = trusted_signers + self.logging = None + self.default_root_object = default_root_object + + def to_xml(self): + s = '\n' + s += '\n' + if self.origin: + s += self.origin.to_xml() + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + if self.trusted_signers: + s += '\n' + for signer in self.trusted_signers: + if signer == 'Self': + s += ' \n' + else: + s += ' %s\n' % signer + s += '\n' + if self.logging: + s += '\n' + s += ' %s\n' % self.logging.bucket + s += ' %s\n' % self.logging.prefix + s += '\n' + if self.default_root_object: + dro = self.default_root_object + s += '%s\n' % dro + s += '\n' + return s + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + elif name == 'Logging': + self.logging = LoggingInfo() + return self.logging + elif name == 'S3Origin': + self.origin = S3Origin() + return self.origin + elif name == 'CustomOrigin': + self.origin = CustomOrigin() + return self.origin + else: + return None + + def endElement(self, name, value, connection): + if name == 'CNAME': + self.cnames.append(value) + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'CallerReference': + self.caller_reference = value + elif name == 'DefaultRootObject': + self.default_root_object = value + else: + setattr(self, name, value) + +class StreamingDistributionConfig(DistributionConfig): + + def __init__(self, connection=None, origin='', enabled=False, + caller_reference='', cnames=None, comment='', + trusted_signers=None, logging=None): + DistributionConfig.__init__(self, connection=connection, + origin=origin, enabled=enabled, + caller_reference=caller_reference, + cnames=cnames, comment=comment, + trusted_signers=trusted_signers, + logging=logging) + def to_xml(self): + s = '\n' + s += '\n' + if self.origin: + s += self.origin.to_xml() + s += ' %s\n' % self.caller_reference + for cname in self.cnames: + s += ' %s\n' % cname + if self.comment: + s += ' %s\n' % self.comment + s += ' ' + if self.enabled: + s += 'true' + else: + s += 'false' + s += '\n' + if self.trusted_signers: + s += '\n' + for signer in self.trusted_signers: + if signer == 'Self': + s += ' \n' + else: + s += ' %s\n' % signer + s += '\n' + if self.logging: + s += '\n' + s += ' %s\n' % self.logging.bucket + s += ' %s\n' % self.logging.prefix + s += '\n' + s += '\n' + return s + +class DistributionSummary: + + def __init__(self, connection=None, domain_name='', id='', + last_modified_time=None, status='', origin=None, + cname='', comment='', enabled=False): + self.connection = connection + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.origin = origin + self.enabled = enabled + self.cnames = [] + if cname: + self.cnames.append(cname) + self.comment = comment + self.trusted_signers = None + self.etag = None + self.streaming = False + + def startElement(self, name, attrs, connection): + if name == 'TrustedSigners': + self.trusted_signers = TrustedSigners() + return self.trusted_signers + elif name == 'S3Origin': + self.origin = S3Origin() + return self.origin + elif name == 'CustomOrigin': + self.origin = CustomOrigin() + return self.origin + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'Status': + self.status = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'DomainName': + self.domain_name = value + elif name == 'Origin': + self.origin = value + elif name == 'CNAME': + self.cnames.append(value) + elif name == 'Comment': + self.comment = value + elif name == 'Enabled': + if value.lower() == 'true': + self.enabled = True + else: + self.enabled = False + elif name == 'StreamingDistributionSummary': + self.streaming = True + else: + setattr(self, name, value) + + def get_distribution(self): + return self.connection.get_distribution_info(self.id) + +class StreamingDistributionSummary(DistributionSummary): + + def get_distribution(self): + return self.connection.get_streaming_distribution_info(self.id) + +class Distribution: + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + self.connection = connection + self.config = config + self.domain_name = domain_name + self.id = id + self.last_modified_time = last_modified_time + self.status = status + self.active_signers = None + self.etag = None + self._bucket = None + self._object_class = Object + + def startElement(self, name, attrs, connection): + if name == 'DistributionConfig': + self.config = DistributionConfig() + return self.config + elif name == 'ActiveTrustedSigners': + self.active_signers = ActiveTrustedSigners() + return self.active_signers + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'LastModifiedTime': + self.last_modified_time = value + elif name == 'Status': + self.status = value + elif name == 'DomainName': + self.domain_name = value + else: + setattr(self, name, value) + + def update(self, enabled=None, cnames=None, comment=None): + """ + Update the configuration of the Distribution. The only values + of the DistributionConfig that can be updated are: + + * CNAMES + * Comment + * Whether the Distribution is enabled or not + + :type enabled: bool + :param enabled: Whether the Distribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + """ + new_config = DistributionConfig(self.connection, self.config.origin, + self.config.enabled, self.config.caller_reference, + self.config.cnames, self.config.comment, + self.config.trusted_signers, + self.config.default_root_object) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + self.etag = self.connection.set_distribution_config(self.id, self.etag, new_config) + self.config = new_config + self._object_class = Object + + def enable(self): + """ + Deactivate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=True) + + def disable(self): + """ + Activate the Distribution. A convenience wrapper around + the update method. + """ + self.update(enabled=False) + + def delete(self): + """ + Delete this CloudFront Distribution. The content + associated with the Distribution is not deleted from + the underlying Origin bucket in S3. + """ + self.connection.delete_distribution(self.id, self.etag) + + def _get_bucket(self): + if not self._bucket: + bucket_name = self.config.origin.replace('.s3.amazonaws.com', '') + from boto.s3.connection import S3Connection + s3 = S3Connection(self.connection.aws_access_key_id, + self.connection.aws_secret_access_key, + proxy=self.connection.proxy, + proxy_port=self.connection.proxy_port, + proxy_user=self.connection.proxy_user, + proxy_pass=self.connection.proxy_pass) + self._bucket = s3.get_bucket(bucket_name) + self._bucket.distribution = self + self._bucket.set_key_class(self._object_class) + return self._bucket + + def get_objects(self): + """ + Return a list of all content objects in this distribution. + + :rtype: list of :class:`boto.cloudfront.object.Object` + :return: The content objects + """ + bucket = self._get_bucket() + objs = [] + for key in bucket: + objs.append(key) + return objs + + def set_permissions(self, object, replace=False): + """ + Sets the S3 ACL grants for the given object to the appropriate + value based on the type of Distribution. If the Distribution + is serving private content the ACL will be set to include the + Origin Access Identity associated with the Distribution. If + the Distribution is serving public content the content will + be set up with "public-read". + + :type object: :class:`boto.cloudfront.object.Object` + :param enabled: The Object whose ACL is being set + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + if isinstance(self.config.origin, S3Origin): + if self.config.origin.origin_access_identity: + id = self.config.origin.origin_access_identity.split('/')[-1] + oai = self.connection.get_origin_access_identity_info(id) + policy = object.get_acl() + if replace: + policy.acl = ACL() + policy.acl.add_user_grant('READ', oai.s3_user_id) + object.set_acl(policy) + else: + object.set_canned_acl('public-read') + + def set_permissions_all(self, replace=False): + """ + Sets the S3 ACL grants for all objects in the Distribution + to the appropriate value based on the type of Distribution. + + :type replace: bool + :param replace: If False, the Origin Access Identity will be + appended to the existing ACL for the object. + If True, the ACL for the object will be + completely replaced with one that grants + READ permission to the Origin Access Identity. + + """ + bucket = self._get_bucket() + for key in bucket: + self.set_permissions(key, replace) + + def add_object(self, name, content, headers=None, replace=True): + """ + Adds a new content object to the Distribution. The content + for the object will be copied to a new Key in the S3 Bucket + and the permissions will be set appropriately for the type + of Distribution. + + :type name: str or unicode + :param name: The name or key of the new object. + + :type content: file-like object + :param content: A file-like object that contains the content + for the new object. + + :type headers: dict + :param headers: A dictionary containing additional headers + you would like associated with the new + object in S3. + + :rtype: :class:`boto.cloudfront.object.Object` + :return: The newly created object. + """ + if self.config.origin_access_identity: + policy = 'private' + else: + policy = 'public-read' + bucket = self._get_bucket() + object = bucket.new_key(name) + object.set_contents_from_file(content, headers=headers, policy=policy) + if self.config.origin_access_identity: + self.set_permissions(object, replace) + return object + +class StreamingDistribution(Distribution): + + def __init__(self, connection=None, config=None, domain_name='', + id='', last_modified_time=None, status=''): + Distribution.__init__(self, connection, config, domain_name, + id, last_modified_time, status) + self._object_class = StreamingObject + + def startElement(self, name, attrs, connection): + if name == 'StreamingDistributionConfig': + self.config = StreamingDistributionConfig() + return self.config + else: + return Distribution.startElement(self, name, attrs, connection) + + def update(self, enabled=None, cnames=None, comment=None): + """ + Update the configuration of the StreamingDistribution. The only values + of the StreamingDistributionConfig that can be updated are: + + * CNAMES + * Comment + * Whether the Distribution is enabled or not + + :type enabled: bool + :param enabled: Whether the StreamingDistribution is active or not. + + :type cnames: list of str + :param cnames: The DNS CNAME's associated with this + Distribution. Maximum of 10 values. + + :type comment: str or unicode + :param comment: The comment associated with the Distribution. + + """ + new_config = StreamingDistributionConfig(self.connection, + self.config.origin, + self.config.enabled, + self.config.caller_reference, + self.config.cnames, + self.config.comment, + self.config.trusted_signers) + if enabled != None: + new_config.enabled = enabled + if cnames != None: + new_config.cnames = cnames + if comment != None: + new_config.comment = comment + self.etag = self.connection.set_streaming_distribution_config(self.id, + self.etag, + new_config) + self.config = new_config + self._object_class = StreamingObject + + def delete(self): + self.connection.delete_streaming_distribution(self.id, self.etag) + + diff --git a/backup/src/boto/cloudfront/exception.py b/backup/src/boto/cloudfront/exception.py new file mode 100644 index 0000000..7680642 --- /dev/null +++ b/backup/src/boto/cloudfront/exception.py @@ -0,0 +1,26 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import BotoServerError + +class CloudFrontServerError(BotoServerError): + + pass diff --git a/backup/src/boto/cloudfront/identity.py b/backup/src/boto/cloudfront/identity.py new file mode 100644 index 0000000..1571e87 --- /dev/null +++ b/backup/src/boto/cloudfront/identity.py @@ -0,0 +1,122 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid + +class OriginAccessIdentity: + + def __init__(self, connection=None, config=None, id='', + s3_user_id='', comment=''): + self.connection = connection + self.config = config + self.id = id + self.s3_user_id = s3_user_id + self.comment = comment + self.etag = None + + def startElement(self, name, attrs, connection): + if name == 'CloudFrontOriginAccessIdentityConfig': + self.config = OriginAccessIdentityConfig() + return self.config + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'S3CanonicalUserId': + self.s3_user_id = value + elif name == 'Comment': + self.comment = value + else: + setattr(self, name, value) + + def update(self, comment=None): + new_config = OriginAccessIdentityConfig(self.connection, + self.config.caller_reference, + self.config.comment) + if comment != None: + new_config.comment = comment + self.etag = self.connection.set_origin_identity_config(self.id, self.etag, new_config) + self.config = new_config + + def delete(self): + return self.connection.delete_origin_access_identity(self.id, self.etag) + + def uri(self): + return 'origin-access-identity/cloudfront/%s' % self.id + +class OriginAccessIdentityConfig: + + def __init__(self, connection=None, caller_reference='', comment=''): + self.connection = connection + if caller_reference: + self.caller_reference = caller_reference + else: + self.caller_reference = str(uuid.uuid4()) + self.comment = comment + + def to_xml(self): + s = '\n' + s += '\n' + s += ' %s\n' % self.caller_reference + if self.comment: + s += ' %s\n' % self.comment + s += '\n' + return s + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Comment': + self.comment = value + elif name == 'CallerReference': + self.caller_reference = value + else: + setattr(self, name, value) + +class OriginAccessIdentitySummary: + + def __init__(self, connection=None, id='', + s3_user_id='', comment=''): + self.connection = connection + self.id = id + self.s3_user_id = s3_user_id + self.comment = comment + self.etag = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'S3CanonicalUserId': + self.s3_user_id = value + elif name == 'Comment': + self.comment = value + else: + setattr(self, name, value) + + def get_origin_access_identity(self): + return self.connection.get_origin_access_identity_info(self.id) + diff --git a/backup/src/boto/cloudfront/invalidation.py b/backup/src/boto/cloudfront/invalidation.py new file mode 100644 index 0000000..ea13a67 --- /dev/null +++ b/backup/src/boto/cloudfront/invalidation.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006-2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import uuid +import urllib + +class InvalidationBatch(object): + """A simple invalidation request. + :see: http://docs.amazonwebservices.com/AmazonCloudFront/2010-08-01/APIReference/index.html?InvalidationBatchDatatype.html + """ + + def __init__(self, paths=[], connection=None, distribution=None, caller_reference=''): + """Create a new invalidation request: + :paths: An array of paths to invalidate + """ + self.paths = paths + self.distribution = distribution + self.caller_reference = caller_reference + if not self.caller_reference: + self.caller_reference = str(uuid.uuid4()) + + # If we passed in a distribution, + # then we use that as the connection object + if distribution: + self.connection = connection + else: + self.connection = connection + + def add(self, path): + """Add another path to this invalidation request""" + return self.paths.append(path) + + def remove(self, path): + """Remove a path from this invalidation request""" + return self.paths.remove(path) + + def __iter__(self): + return iter(self.paths) + + def __getitem__(self, i): + return self.paths[i] + + def __setitem__(self, k, v): + self.paths[k] = v + + def escape(self, p): + """Escape a path, make sure it begins with a slash and contains no invalid characters""" + if not p[0] == "/": + p = "/%s" % p + return urllib.quote(p) + + def to_xml(self): + """Get this batch as XML""" + assert self.connection != None + s = '\n' + s += '\n' % self.connection.Version + for p in self.paths: + s += ' %s\n' % self.escape(p) + s += ' %s\n' % self.caller_reference + s += '\n' + return s + + def startElement(self, name, attrs, connection): + if name == "InvalidationBatch": + self.paths = [] + return None + + def endElement(self, name, value, connection): + if name == 'Path': + self.paths.append(value) + elif name == "Status": + self.status = value + elif name == "Id": + self.id = value + elif name == "CreateTime": + self.create_time = value + elif name == "CallerReference": + self.caller_reference = value + return None diff --git a/backup/src/boto/cloudfront/logging.py b/backup/src/boto/cloudfront/logging.py new file mode 100644 index 0000000..6c2f4fd --- /dev/null +++ b/backup/src/boto/cloudfront/logging.py @@ -0,0 +1,38 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class LoggingInfo(object): + + def __init__(self, bucket='', prefix=''): + self.bucket = bucket + self.prefix = prefix + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Bucket': + self.bucket = value + elif name == 'Prefix': + self.prefix = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/cloudfront/object.py b/backup/src/boto/cloudfront/object.py new file mode 100644 index 0000000..3574d13 --- /dev/null +++ b/backup/src/boto/cloudfront/object.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.key import Key + +class Object(Key): + + def __init__(self, bucket, name=None): + Key.__init__(self, bucket, name=name) + self.distribution = bucket.distribution + + def __repr__(self): + return '' % (self.distribution.config.origin, self.name) + + def url(self, scheme='http'): + url = '%s://' % scheme + url += self.distribution.domain_name + if scheme.lower().startswith('rtmp'): + url += '/cfx/st/' + else: + url += '/' + url += self.name + return url + +class StreamingObject(Object): + + def url(self, scheme='rtmp'): + return Object.url(self, scheme) + + diff --git a/backup/src/boto/cloudfront/origin.py b/backup/src/boto/cloudfront/origin.py new file mode 100644 index 0000000..57af846 --- /dev/null +++ b/backup/src/boto/cloudfront/origin.py @@ -0,0 +1,150 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from identity import OriginAccessIdentity + +def get_oai_value(origin_access_identity): + if isinstance(origin_access_identity, OriginAccessIdentity): + return origin_access_identity.uri() + else: + return origin_access_identity + +class S3Origin(object): + """ + Origin information to associate with the distribution. + If your distribution will use an Amazon S3 origin, + then you use the S3Origin element. + """ + + def __init__(self, dns_name=None, origin_access_identity=None): + """ + :param dns_name: The DNS name of your Amazon S3 bucket to + associate with the distribution. + For example: mybucket.s3.amazonaws.com. + :type dns_name: str + + :param origin_access_identity: The CloudFront origin access + identity to associate with the + distribution. If you want the + distribution to serve private content, + include this element; if you want the + distribution to serve public content, + remove this element. + :type origin_access_identity: str + + """ + self.dns_name = dns_name + self.origin_access_identity = origin_access_identity + + def __repr__(self): + return '' % self.dns_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DNSName': + self.dns_name = value + elif name == 'OriginAccessIdentity': + self.origin_access_identity = value + else: + setattr(self, name, value) + + def to_xml(self): + s = ' \n' + s += ' %s\n' % self.dns_name + if self.origin_access_identity: + val = get_oai_value(self.origin_access_identity) + s += ' %s\n' % val + s += ' \n' + return s + +class CustomOrigin(object): + """ + Origin information to associate with the distribution. + If your distribution will use a non-Amazon S3 origin, + then you use the CustomOrigin element. + """ + + def __init__(self, dns_name=None, http_port=80, https_port=443, + origin_protocol_policy=None): + """ + :param dns_name: The DNS name of your Amazon S3 bucket to + associate with the distribution. + For example: mybucket.s3.amazonaws.com. + :type dns_name: str + + :param http_port: The HTTP port the custom origin listens on. + :type http_port: int + + :param https_port: The HTTPS port the custom origin listens on. + :type http_port: int + + :param origin_protocol_policy: The origin protocol policy to + apply to your origin. If you + specify http-only, CloudFront + will use HTTP only to access the origin. + If you specify match-viewer, CloudFront + will fetch from your origin using HTTP + or HTTPS, based on the protocol of the + viewer request. + :type origin_protocol_policy: str + + """ + self.dns_name = dns_name + self.http_port = http_port + self.https_port = https_port + self.origin_protocol_policy = origin_protocol_policy + + def __repr__(self): + return '' % self.dns_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DNSName': + self.dns_name = value + elif name == 'HTTPPort': + try: + self.http_port = int(value) + except ValueError: + self.http_port = value + elif name == 'HTTPSPort': + try: + self.https_port = int(value) + except ValueError: + self.https_port = value + elif name == 'OriginProtocolPolicy': + self.origin_protocol_policy = value + else: + setattr(self, name, value) + + def to_xml(self): + s = ' \n' + s += ' %s\n' % self.dns_name + s += ' %d\n' % self.http_port + s += ' %d\n' % self.https_port + s += ' %s\n' % self.origin_protocol_policy + s += ' \n' + return s + diff --git a/backup/src/boto/cloudfront/signers.py b/backup/src/boto/cloudfront/signers.py new file mode 100644 index 0000000..0b0cd50 --- /dev/null +++ b/backup/src/boto/cloudfront/signers.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Signer: + + def __init__(self): + self.id = None + self.key_pair_ids = [] + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.id = 'Self' + elif name == 'AwsAccountNumber': + self.id = value + elif name == 'KeyPairId': + self.key_pair_ids.append(value) + +class ActiveTrustedSigners(list): + + def startElement(self, name, attrs, connection): + if name == 'Signer': + s = Signer() + self.append(s) + return s + + def endElement(self, name, value, connection): + pass + +class TrustedSigners(list): + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Self': + self.append(name) + elif name == 'AwsAccountNumber': + self.append(value) + diff --git a/backup/src/boto/connection.py b/backup/src/boto/connection.py new file mode 100644 index 0000000..76e9ffe --- /dev/null +++ b/backup/src/boto/connection.py @@ -0,0 +1,637 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010 Google +# Copyright (c) 2008 rPath, Inc. +# Copyright (c) 2009 The Echo Nest Corporation +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Handles basic connections to AWS +""" + +import base64 +import errno +import httplib +import os +import Queue +import re +import socket +import sys +import time +import urllib, urlparse +import xml.sax + +import auth +import auth_handler +import boto +import boto.utils + +from boto import config, UserAgent, handler +from boto.exception import AWSConnectionError, BotoClientError, BotoServerError +from boto.provider import Provider +from boto.resultset import ResultSet + + +PORTS_BY_SECURITY = { True: 443, False: 80 } + +class ConnectionPool: + def __init__(self, hosts, connections_per_host): + self._hosts = boto.utils.LRUCache(hosts) + self.connections_per_host = connections_per_host + + def __getitem__(self, key): + if key not in self._hosts: + self._hosts[key] = Queue.Queue(self.connections_per_host) + return self._hosts[key] + + def __repr__(self): + return 'ConnectionPool:%s' % ','.join(self._hosts._dict.keys()) + +class HTTPRequest(object): + + def __init__(self, method, protocol, host, port, path, auth_path, + params, headers, body): + """Represents an HTTP request. + + :type method: string + :param method: The HTTP method name, 'GET', 'POST', 'PUT' etc. + + :type protocol: string + :param protocol: The http protocol used, 'http' or 'https'. + + :type host: string + :param host: Host to which the request is addressed. eg. abc.com + + :type port: int + :param port: port on which the request is being sent. Zero means unset, + in which case default port will be chosen. + + :type path: string + :param path: URL path that is bein accessed. + + :type auth_path: string + :param path: The part of the URL path used when creating the + authentication string. + + :type params: dict + :param params: HTTP url query parameters, with key as name of the param, + and value as value of param. + + :type headers: dict + :param headers: HTTP headers, with key as name of the header and value + as value of header. + + :type body: string + :param body: Body of the HTTP request. If not present, will be None or + empty string (''). + """ + self.method = method + self.protocol = protocol + self.host = host + self.port = port + self.path = path + self.auth_path = auth_path + self.params = params + self.headers = headers + self.body = body + + def __str__(self): + return (('method:(%s) protocol:(%s) host(%s) port(%s) path(%s) ' + 'params(%s) headers(%s) body(%s)') % (self.method, + self.protocol, self.host, self.port, self.path, self.params, + self.headers, self.body)) + +class AWSAuthConnection(object): + def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, path='/', provider='aws'): + """ + :type host: str + :param host: The host to make the connection to + + :keyword str aws_access_key_id: Your AWS Access Key ID (provided by + Amazon). If none is specified, the value in your + ``AWS_ACCESS_KEY_ID`` environmental variable is used. + :keyword str aws_secret_access_key: Your AWS Secret Access Key + (provided by Amazon). If none is specified, the value in your + ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. + + :type is_secure: boolean + :param is_secure: Whether the connection is over SSL + + :type https_connection_factory: list or tuple + :param https_connection_factory: A pair of an HTTP connection + factory and the exceptions to catch. + The factory should have a similar + interface to L{httplib.HTTPSConnection}. + + :param str proxy: Address/hostname for a proxy server + + :type proxy_port: int + :param proxy_port: The port to use when connecting over a proxy + + :type proxy_user: str + :param proxy_user: The username to connect with on the proxy + + :type proxy_pass: str + :param proxy_pass: The password to use when connection over a proxy. + + :type port: int + :param port: The port to use to connect + """ + self.num_retries = 5 + # Override passed-in is_secure setting if value was defined in config. + if config.has_option('Boto', 'is_secure'): + is_secure = config.getboolean('Boto', 'is_secure') + self.is_secure = is_secure + self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass) + # define exceptions from httplib that we want to catch and retry + self.http_exceptions = (httplib.HTTPException, socket.error, + socket.gaierror) + # define values in socket exceptions we don't want to catch + self.socket_exception_values = (errno.EINTR,) + if https_connection_factory is not None: + self.https_connection_factory = https_connection_factory[0] + self.http_exceptions += https_connection_factory[1] + else: + self.https_connection_factory = None + if (is_secure): + self.protocol = 'https' + else: + self.protocol = 'http' + self.host = host + self.path = path + if debug: + self.debug = debug + else: + self.debug = config.getint('Boto', 'debug', debug) + if port: + self.port = port + else: + self.port = PORTS_BY_SECURITY[is_secure] + + self.provider = Provider(provider, + aws_access_key_id, + aws_secret_access_key) + + # allow config file to override default host + if self.provider.host: + self.host = self.provider.host + + # cache up to 20 connections per host, up to 20 hosts + self._pool = ConnectionPool(20, 20) + self._connection = (self.server_name(), self.is_secure) + self._last_rs = None + self._auth_handler = auth.get_auth_handler( + host, config, self.provider, self._required_auth_capability()) + + def __repr__(self): + return '%s:%s' % (self.__class__.__name__, self.host) + + def _required_auth_capability(self): + return [] + + def _cached_name(self, host, is_secure): + if host is None: + host = self.server_name() + cached_name = is_secure and 'https://' or 'http://' + cached_name += host + return cached_name + + def connection(self): + return self.get_http_connection(*self._connection) + connection = property(connection) + + def aws_access_key_id(self): + return self.provider.access_key + aws_access_key_id = property(aws_access_key_id) + gs_access_key_id = aws_access_key_id + access_key = aws_access_key_id + + def aws_secret_access_key(self): + return self.provider.secret_key + aws_secret_access_key = property(aws_secret_access_key) + gs_secret_access_key = aws_secret_access_key + secret_key = aws_secret_access_key + + def get_path(self, path='/'): + pos = path.find('?') + if pos >= 0: + params = path[pos:] + path = path[:pos] + else: + params = None + if path[-1] == '/': + need_trailing = True + else: + need_trailing = False + path_elements = self.path.split('/') + path_elements.extend(path.split('/')) + path_elements = [p for p in path_elements if p] + path = '/' + '/'.join(path_elements) + if path[-1] != '/' and need_trailing: + path += '/' + if params: + path = path + params + return path + + def server_name(self, port=None): + if not port: + port = self.port + if port == 80: + signature_host = self.host + else: + # This unfortunate little hack can be attributed to + # a difference in the 2.6 version of httplib. In old + # versions, it would append ":443" to the hostname sent + # in the Host header and so we needed to make sure we + # did the same when calculating the V2 signature. In 2.6 + # (and higher!) + # it no longer does that. Hence, this kludge. + if sys.version[:3] in ('2.6', '2.7') and port == 443: + signature_host = self.host + else: + signature_host = '%s:%d' % (self.host, port) + return signature_host + + def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass): + self.proxy = proxy + self.proxy_port = proxy_port + self.proxy_user = proxy_user + self.proxy_pass = proxy_pass + if os.environ.has_key('http_proxy') and not self.proxy: + pattern = re.compile( + '(?:http://)?' \ + '(?:(?P\w+):(?P.*)@)?' \ + '(?P[\w\-\.]+)' \ + '(?::(?P\d+))?' + ) + match = pattern.match(os.environ['http_proxy']) + if match: + self.proxy = match.group('host') + self.proxy_port = match.group('port') + self.proxy_user = match.group('user') + self.proxy_pass = match.group('pass') + else: + if not self.proxy: + self.proxy = config.get_value('Boto', 'proxy', None) + if not self.proxy_port: + self.proxy_port = config.get_value('Boto', 'proxy_port', None) + if not self.proxy_user: + self.proxy_user = config.get_value('Boto', 'proxy_user', None) + if not self.proxy_pass: + self.proxy_pass = config.get_value('Boto', 'proxy_pass', None) + + if not self.proxy_port and self.proxy: + print "http_proxy environment variable does not specify " \ + "a port, using default" + self.proxy_port = self.port + self.use_proxy = (self.proxy != None) + + def get_http_connection(self, host, is_secure): + queue = self._pool[self._cached_name(host, is_secure)] + try: + return queue.get_nowait() + except Queue.Empty: + return self.new_http_connection(host, is_secure) + + def new_http_connection(self, host, is_secure): + if self.use_proxy: + host = '%s:%d' % (self.proxy, int(self.proxy_port)) + if host is None: + host = self.server_name() + if is_secure: + boto.log.debug('establishing HTTPS connection') + if self.use_proxy: + connection = self.proxy_ssl() + elif self.https_connection_factory: + connection = self.https_connection_factory(host) + else: + connection = httplib.HTTPSConnection(host) + else: + boto.log.debug('establishing HTTP connection') + connection = httplib.HTTPConnection(host) + if self.debug > 1: + connection.set_debuglevel(self.debug) + # self.connection must be maintained for backwards-compatibility + # however, it must be dynamically pulled from the connection pool + # set a private variable which will enable that + if host.split(':')[0] == self.host and is_secure == self.is_secure: + self._connection = (host, is_secure) + return connection + + def put_http_connection(self, host, is_secure, connection): + try: + self._pool[self._cached_name(host, is_secure)].put_nowait(connection) + except Queue.Full: + # gracefully fail in case of pool overflow + connection.close() + + def proxy_ssl(self): + host = '%s:%d' % (self.host, self.port) + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + try: + sock.connect((self.proxy, int(self.proxy_port))) + except: + raise + sock.sendall("CONNECT %s HTTP/1.0\r\n" % host) + sock.sendall("User-Agent: %s\r\n" % UserAgent) + if self.proxy_user and self.proxy_pass: + for k, v in self.get_proxy_auth_header().items(): + sock.sendall("%s: %s\r\n" % (k, v)) + sock.sendall("\r\n") + resp = httplib.HTTPResponse(sock, strict=True) + resp.begin() + + if resp.status != 200: + # Fake a socket error, use a code that make it obvious it hasn't + # been generated by the socket library + raise socket.error(-71, + "Error talking to HTTP proxy %s:%s: %s (%s)" % + (self.proxy, self.proxy_port, resp.status, resp.reason)) + + # We can safely close the response, it duped the original socket + resp.close() + + h = httplib.HTTPConnection(host) + + # Wrap the socket in an SSL socket + if hasattr(httplib, 'ssl'): + sslSock = httplib.ssl.SSLSocket(sock) + else: # Old Python, no ssl module + sslSock = socket.ssl(sock, None, None) + sslSock = httplib.FakeSocket(sock, sslSock) + # This is a bit unclean + h.sock = sslSock + return h + + def prefix_proxy_to_path(self, path, host=None): + path = self.protocol + '://' + (host or self.server_name()) + path + return path + + def get_proxy_auth_header(self): + auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass) + return {'Proxy-Authorization': 'Basic %s' % auth} + + def _mexe(self, method, path, data, headers, host=None, sender=None, + override_num_retries=None): + """ + mexe - Multi-execute inside a loop, retrying multiple times to handle + transient Internet errors by simply trying again. + Also handles redirects. + + This code was inspired by the S3Utils classes posted to the boto-users + Google group by Larry Bates. Thanks! + """ + boto.log.debug('Method: %s' % method) + boto.log.debug('Path: %s' % path) + boto.log.debug('Data: %s' % data) + boto.log.debug('Headers: %s' % headers) + boto.log.debug('Host: %s' % host) + response = None + body = None + e = None + if override_num_retries is None: + num_retries = config.getint('Boto', 'num_retries', self.num_retries) + else: + num_retries = override_num_retries + i = 0 + connection = self.get_http_connection(host, self.is_secure) + while i <= num_retries: + try: + if callable(sender): + response = sender(connection, method, path, data, headers) + else: + connection.request(method, path, data, headers) + response = connection.getresponse() + location = response.getheader('location') + # -- gross hack -- + # httplib gets confused with chunked responses to HEAD requests + # so I have to fake it out + if method == 'HEAD' and getattr(response, 'chunked', False): + response.chunked = 0 + if response.status == 500 or response.status == 503: + boto.log.debug('received %d response, retrying in %d seconds' % (response.status, 2 ** i)) + body = response.read() + elif response.status == 408: + body = response.read() + print '-------------------------' + print ' 4 0 8 ' + print 'path=%s' % path + print body + print '-------------------------' + elif response.status < 300 or response.status >= 400 or \ + not location: + self.put_http_connection(host, self.is_secure, connection) + return response + else: + scheme, host, path, params, query, fragment = \ + urlparse.urlparse(location) + if query: + path += '?' + query + boto.log.debug('Redirecting: %s' % scheme + '://' + host + path) + connection = self.get_http_connection(host, scheme == 'https') + continue + except KeyboardInterrupt: + sys.exit('Keyboard Interrupt') + except self.http_exceptions, e: + boto.log.debug('encountered %s exception, reconnecting' % \ + e.__class__.__name__) + connection = self.new_http_connection(host, self.is_secure) + time.sleep(2 ** i) + i += 1 + # If we made it here, it's because we have exhausted our retries and stil haven't + # succeeded. So, if we have a response object, use it to raise an exception. + # Otherwise, raise the exception that must have already happened. + if response: + raise BotoServerError(response.status, response.reason, body) + elif e: + raise e + else: + raise BotoClientError('Please report this exception as a Boto Issue!') + + def build_base_http_request(self, method, path, auth_path, + params=None, headers=None, data='', host=None): + path = self.get_path(path) + if auth_path is not None: + auth_path = self.get_path(auth_path) + if params == None: + params = {} + else: + params = params.copy() + if headers == None: + headers = {} + else: + headers = headers.copy() + host = host or self.host + if self.use_proxy: + path = self.prefix_proxy_to_path(path, host) + if self.proxy_user and self.proxy_pass and not self.is_secure: + # If is_secure, we don't have to set the proxy authentication + # header here, we did that in the CONNECT to the proxy. + headers.update(self.get_proxy_auth_header()) + return HTTPRequest(method, self.protocol, host, self.port, + path, auth_path, params, headers, data) + + def fill_in_auth(self, http_request, **kwargs): + headers = http_request.headers + for key in headers: + val = headers[key] + if isinstance(val, unicode): + headers[key] = urllib.quote_plus(val.encode('utf-8')) + + self._auth_handler.add_auth(http_request, **kwargs) + + headers['User-Agent'] = UserAgent + if not headers.has_key('Content-Length'): + headers['Content-Length'] = str(len(http_request.body)) + return http_request + + def _send_http_request(self, http_request, sender=None, + override_num_retries=None): + return self._mexe(http_request.method, http_request.path, + http_request.body, http_request.headers, + http_request.host, sender, override_num_retries) + + def make_request(self, method, path, headers=None, data='', host=None, + auth_path=None, sender=None, override_num_retries=None): + """Makes a request to the server, with stock multiple-retry logic.""" + http_request = self.build_base_http_request(method, path, auth_path, + {}, headers, data, host) + http_request = self.fill_in_auth(http_request) + return self._send_http_request(http_request, sender, + override_num_retries) + + def close(self): + """(Optional) Close any open HTTP connections. This is non-destructive, + and making a new request will open a connection again.""" + + boto.log.debug('closing all HTTP connections') + self.connection = None # compat field + +class AWSQueryConnection(AWSAuthConnection): + + APIVersion = '' + ResponseError = BotoServerError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host=None, debug=0, + https_connection_factory=None, path='/'): + AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug, https_connection_factory, path) + + def _required_auth_capability(self): + return [] + + def get_utf8_value(self, value): + return boto.utils.get_utf8_value(value) + + def make_request(self, action, params=None, path='/', verb='GET'): + http_request = self.build_base_http_request(verb, path, None, + params, {}, '', + self.server_name()) + if action: + http_request.params['Action'] = action + http_request.params['Version'] = self.APIVersion + http_request = self.fill_in_auth(http_request) + return self._send_http_request(http_request) + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items) + 1): + params['%s.%d' % (label, i)] = items[i - 1] + + # generics + + def get_list(self, action, params, markers, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + rs = ResultSet(markers) + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_object(self, action, params, cls, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + obj = cls(parent) + h = handler.XmlHandler(obj, parent) + xml.sax.parseString(body, h) + return obj + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_status(self, action, params, path='/', parent=None, verb='GET'): + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if not body: + boto.log.error('Null body %s' % body) + raise self.ResponseError(response.status, response.reason, body) + elif response.status == 200: + rs = ResultSet() + h = handler.XmlHandler(rs, parent) + xml.sax.parseString(body, h) + return rs.status + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) diff --git a/backup/src/boto/contrib/__init__.py b/backup/src/boto/contrib/__init__.py new file mode 100644 index 0000000..303dbb6 --- /dev/null +++ b/backup/src/boto/contrib/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/backup/src/boto/contrib/m2helpers.py b/backup/src/boto/contrib/m2helpers.py new file mode 100644 index 0000000..82d2730 --- /dev/null +++ b/backup/src/boto/contrib/m2helpers.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Jon Colverson +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Jon Colverson. It provides a couple of helper +functions that allow you to use M2Crypto's implementation of HTTPSConnection +rather than the default version in httplib.py. The main benefit is that +M2Crypto's version verifies the certificate of the server. + +To use this feature, do something like this: + +from boto.ec2.connection import EC2Connection + +ec2 = EC2Connection(ACCESS_KEY_ID, SECRET_ACCESS_KEY, + https_connection_factory=https_connection_factory(cafile=CA_FILE)) + +See http://code.google.com/p/boto/issues/detail?id=57 for more details. +""" +from M2Crypto import SSL +from M2Crypto.httpslib import HTTPSConnection + +def secure_context(cafile=None, capath=None): + ctx = SSL.Context() + ctx.set_verify(SSL.verify_peer | SSL.verify_fail_if_no_peer_cert, depth=9) + if ctx.load_verify_locations(cafile=cafile, capath=capath) != 1: + raise Exception("Couldn't load certificates") + return ctx + +def https_connection_factory(cafile=None, capath=None): + def factory(*args, **kwargs): + return HTTPSConnection( + ssl_context=secure_context(cafile=cafile, capath=capath), + *args, **kwargs) + return (factory, (SSL.SSLError,)) diff --git a/backup/src/boto/contrib/ymlmessage.py b/backup/src/boto/contrib/ymlmessage.py new file mode 100644 index 0000000..b9a2c93 --- /dev/null +++ b/backup/src/boto/contrib/ymlmessage.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006,2007 Chris Moyer +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module was contributed by Chris Moyer. It provides a subclass of the +SQS Message class that supports YAML as the body of the message. + +This module requires the yaml module. +""" +from boto.sqs.message import Message +import yaml + +class YAMLMessage(Message): + """ + The YAMLMessage class provides a YAML compatible message. Encoding and + decoding are handled automaticaly. + + Access this message data like such: + + m.data = [ 1, 2, 3] + m.data[0] # Returns 1 + + This depends on the PyYAML package + """ + + def __init__(self, queue=None, body='', xml_attrs=None): + self.data = None + Message.__init__(self, queue, body) + + def set_body(self, body): + self.data = yaml.load(body) + + def get_body(self): + return yaml.dump(self.data) diff --git a/backup/src/boto/ec2/__init__.py b/backup/src/boto/ec2/__init__.py new file mode 100644 index 0000000..8bb3f53 --- /dev/null +++ b/backup/src/boto/ec2/__init__.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +service from AWS. +""" +from boto.ec2.connection import EC2Connection + +def regions(**kw_params): + """ + Get all available regions for the EC2 service. + You may pass any of the arguments accepted by the EC2Connection + object's constructor as keyword arguments and they will be + passed along to the EC2Connection object. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + c = EC2Connection(**kw_params) + return c.get_all_regions() + +def connect_to_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region.connect(**kw_params) + return None + +def get_region(region_name, **kw_params): + for region in regions(**kw_params): + if region.name == region_name: + return region + return None + diff --git a/backup/src/boto/ec2/address.py b/backup/src/boto/ec2/address.py new file mode 100644 index 0000000..60ed406 --- /dev/null +++ b/backup/src/boto/ec2/address.py @@ -0,0 +1,58 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Address +""" + +from boto.ec2.ec2object import EC2Object + +class Address(EC2Object): + + def __init__(self, connection=None, public_ip=None, instance_id=None): + EC2Object.__init__(self, connection) + self.connection = connection + self.public_ip = public_ip + self.instance_id = instance_id + + def __repr__(self): + return 'Address:%s' % self.public_ip + + def endElement(self, name, value, connection): + if name == 'publicIp': + self.public_ip = value + elif name == 'instanceId': + self.instance_id = value + else: + setattr(self, name, value) + + def release(self): + return self.connection.release_address(self.public_ip) + + delete = release + + def associate(self, instance_id): + return self.connection.associate_address(instance_id, self.public_ip) + + def disassociate(self): + return self.connection.disassociate_address(self.public_ip) + + diff --git a/backup/src/boto/ec2/autoscale/__init__.py b/backup/src/boto/ec2/autoscale/__init__.py new file mode 100644 index 0000000..e8dd695 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/__init__.py @@ -0,0 +1,282 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +Auto Scaling service. +""" + +import base64 +import boto +from boto.connection import AWSQueryConnection +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.autoscale.request import Request +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.launchconfig import LaunchConfiguration +from boto.ec2.autoscale.group import AutoScalingGroup +from boto.ec2.autoscale.activity import Activity + +RegionData = { + 'us-east-1' : 'autoscaling.us-east-1.amazonaws.com', + 'us-west-1' : 'autoscaling.us-west-1.amazonaws.com', + 'eu-west-1' : 'autoscaling.eu-west-1.amazonaws.com', + 'ap-southeast-1' : 'autoscaling.ap-southeast-1.amazonaws.com'} + +def regions(): + """ + Get all available regions for the Auto Scaling service. + + :rtype: list + :return: A list of :class:`boto.RegionInfo` instances + """ + regions = [] + for region_name in RegionData: + region = RegionInfo(name=region_name, + endpoint=RegionData[region_name], + connection_cls=AutoScaleConnection) + regions.append(region) + return regions + +def connect_to_region(region_name, **kw_params): + """ + Given a valid region name, return a + :class:`boto.ec2.autoscale.AutoScaleConnection`. + + :param str region_name: The name of the region to connect to. + + :rtype: :class:`boto.ec2.AutoScaleConnection` or ``None`` + :return: A connection to the given region, or None if an invalid region + name is given + """ + for region in regions(): + if region.name == region_name: + return region.connect(**kw_params) + return None + + +class AutoScaleConnection(AWSQueryConnection): + APIVersion = boto.config.get('Boto', 'autoscale_version', '2010-08-01') + Endpoint = boto.config.get('Boto', 'autoscale_endpoint', + 'autoscaling.amazonaws.com') + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'autoscaling.amazonaws.com' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=1, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to the AutoScaling service. + + B{Note:} The host argument is overridden by the host specified in the + boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint, + AutoScaleConnection) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path=path) + + def _required_auth_capability(self): + return ['ec2'] + + def build_list_params(self, params, items, label): + """ items is a list of dictionaries or strings: + [{'Protocol' : 'HTTP', + 'LoadBalancerPort' : '80', + 'InstancePort' : '80'},..] etc. + or + ['us-east-1b',...] + """ + # different from EC2 list params + for i in xrange(1, len(items)+1): + if isinstance(items[i-1], dict): + for k, v in items[i-1].iteritems(): + params['%s.member.%d.%s' % (label, i, k)] = v + elif isinstance(items[i-1], basestring): + params['%s.member.%d' % (label, i)] = items[i-1] + + def _update_group(self, op, as_group): + params = { + 'AutoScalingGroupName' : as_group.name, + 'Cooldown' : as_group.cooldown, + 'LaunchConfigurationName' : as_group.launch_config_name, + 'MinSize' : as_group.min_size, + 'MaxSize' : as_group.max_size, + } + if op.startswith('Create'): + if as_group.availability_zones: + zones = as_group.availability_zones + else: + zones = [as_group.availability_zone] + self.build_list_params(params, as_group.load_balancers, + 'LoadBalancerNames') + self.build_list_params(params, zones, + 'AvailabilityZones') + return self.get_object(op, params, Request) + + def create_auto_scaling_group(self, as_group): + """ + Create auto scaling group. + """ + return self._update_group('CreateAutoScalingGroup', as_group) + + def create_launch_configuration(self, launch_config): + """ + Creates a new Launch Configuration. + + :type launch_config: boto.ec2.autoscale.launchconfig.LaunchConfiguration + :param launch_config: LaunchConfiguraiton object. + + """ + params = { + 'ImageId' : launch_config.image_id, + 'KeyName' : launch_config.key_name, + 'LaunchConfigurationName' : launch_config.name, + 'InstanceType' : launch_config.instance_type, + } + if launch_config.user_data: + params['UserData'] = base64.b64encode(launch_config.user_data) + if launch_config.kernel_id: + params['KernelId'] = launch_config.kernel_id + if launch_config.ramdisk_id: + params['RamdiskId'] = launch_config.ramdisk_id + if launch_config.block_device_mappings: + self.build_list_params(params, launch_config.block_device_mappings, + 'BlockDeviceMappings') + self.build_list_params(params, launch_config.security_groups, + 'SecurityGroups') + return self.get_object('CreateLaunchConfiguration', params, + Request, verb='POST') + + def create_trigger(self, trigger): + """ + + """ + params = {'TriggerName' : trigger.name, + 'AutoScalingGroupName' : trigger.autoscale_group.name, + 'MeasureName' : trigger.measure_name, + 'Statistic' : trigger.statistic, + 'Period' : trigger.period, + 'Unit' : trigger.unit, + 'LowerThreshold' : trigger.lower_threshold, + 'LowerBreachScaleIncrement' : trigger.lower_breach_scale_increment, + 'UpperThreshold' : trigger.upper_threshold, + 'UpperBreachScaleIncrement' : trigger.upper_breach_scale_increment, + 'BreachDuration' : trigger.breach_duration} + # dimensions should be a list of tuples + dimensions = [] + for dim in trigger.dimensions: + name, value = dim + dimensions.append(dict(Name=name, Value=value)) + self.build_list_params(params, dimensions, 'Dimensions') + + req = self.get_object('CreateOrUpdateScalingTrigger', params, + Request) + return req + + def get_all_groups(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'AutoScalingGroupNames') + return self.get_list('DescribeAutoScalingGroups', params, + [('member', AutoScalingGroup)]) + + def get_all_launch_configurations(self, names=None): + """ + """ + params = {} + if names: + self.build_list_params(params, names, 'LaunchConfigurationNames') + return self.get_list('DescribeLaunchConfigurations', params, + [('member', LaunchConfiguration)]) + + def get_all_activities(self, autoscale_group, + activity_ids=None, + max_records=100): + """ + Get all activities for the given autoscaling group. + + :type autoscale_group: str or AutoScalingGroup object + :param autoscale_group: The auto scaling group to get activities on. + + @max_records: int + :param max_records: Maximum amount of activities to return. + """ + name = autoscale_group + if isinstance(autoscale_group, AutoScalingGroup): + name = autoscale_group.name + params = {'AutoScalingGroupName' : name} + if activity_ids: + self.build_list_params(params, activity_ids, 'ActivityIds') + return self.get_list('DescribeScalingActivities', params, + [('member', Activity)]) + + def get_all_triggers(self, autoscale_group): + params = {'AutoScalingGroupName' : autoscale_group} + return self.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + def terminate_instance(self, instance_id, decrement_capacity=True): + params = { + 'InstanceId' : instance_id, + 'ShouldDecrementDesiredCapacity' : decrement_capacity + } + return self.get_object('TerminateInstanceInAutoScalingGroup', params, + Activity) + + def set_instance_health(self, instance_id, health_status, + should_respect_grace_period=True): + """ + Explicitly set the health status of an instance. + + :type instance_id: str + :param instance_id: The identifier of the EC2 instance. + + :type health_status: str + :param health_status: The health status of the instance. + "Healthy" means that the instance is + healthy and should remain in service. + "Unhealthy" means that the instance is + unhealthy. Auto Scaling should terminate + and replace it. + + :type should_respect_grace_period: bool + :param should_respect_grace_period: If True, this call should + respect the grace period + associated with the group. + """ + params = {'InstanceId' : instance_id, + 'HealthStatus' : health_status} + if should_respect_grace_period: + params['ShouldRespectGracePeriod'] = 'true' + else: + params['ShouldRespectGracePeriod'] = 'false' + return self.get_status('SetInstanceHealth', params) + diff --git a/backup/src/boto/ec2/autoscale/activity.py b/backup/src/boto/ec2/autoscale/activity.py new file mode 100644 index 0000000..f895d65 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/activity.py @@ -0,0 +1,55 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Activity(object): + def __init__(self, connection=None): + self.connection = connection + self.start_time = None + self.activity_id = None + self.progress = None + self.status_code = None + self.cause = None + self.description = None + + def __repr__(self): + return 'Activity:%s status:%s progress:%s' % (self.description, + self.status_code, + self.progress) + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ActivityId': + self.activity_id = value + elif name == 'StartTime': + self.start_time = value + elif name == 'Progress': + self.progress = value + elif name == 'Cause': + self.cause = value + elif name == 'Description': + self.description = value + elif name == 'StatusCode': + self.status_code = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/ec2/autoscale/group.py b/backup/src/boto/ec2/autoscale/group.py new file mode 100644 index 0000000..9010a72 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/group.py @@ -0,0 +1,235 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref +from boto.ec2.elb.listelement import ListElement +from boto.resultset import ResultSet +from boto.ec2.autoscale.trigger import Trigger +from boto.ec2.autoscale.request import Request + + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + else: + setattr(self, name, value) + + +class AutoScalingGroup(object): + def __init__(self, connection=None, group_name=None, + availability_zone=None, launch_config=None, + availability_zones=None, + load_balancers=None, cooldown=0, + min_size=None, max_size=None, + group_arn=None, health_check_type=None, + health_check_period=None, suspended=None, + placement_group=None, vpc_zone=None): + """ + Creates a new AutoScalingGroup with the specified name. + + You must not have already used up your entire quota of + AutoScalingGroups in order for this call to be successful. Once the + creation request is completed, the AutoScalingGroup is ready to be + used in other calls. + + :type name: str + :param name: Name of autoscaling group. + + :type availability_zone: str + :param availability_zone: An availability zone. DEPRECATED - use the + availability_zones parameter, which expects + a list of availability zone + strings + + :type availability_zone: list + :param availability_zone: List of availability zones. + + :type launch_config: str + :param launch_config: Name of launch configuration name. + + :type load_balancers: list + :param load_balancers: List of load balancers. + + :type minsize: int + :param minsize: Minimum size of group + + :type maxsize: int + :param maxsize: Maximum size of group + + :type cooldown: int + :param cooldown: Amount of time after a Scaling Activity completes + before any further scaling activities can start. + + :rtype: tuple + :return: Updated healthcheck for the instances. + """ + self.name = group_name + self.connection = connection + self.min_size = min_size + self.max_size = max_size + self.created_time = None + self.cooldown = cooldown + self.launch_config = launch_config + if self.launch_config: + self.launch_config_name = self.launch_config.name + else: + self.launch_config_name = None + self.desired_capacity = None + lbs = load_balancers or [] + self.load_balancers = ListElement(lbs) + zones = availability_zones or [] + self.availability_zone = availability_zone + self.availability_zones = ListElement(zones) + self.group_arn = group_arn + self.health_check_type = health_check_type + self.health_check_period = health_check_period + self.suspended = suspended + self.placement_group = placement_group + self.vpc_zone = vpc_zone + self.metrics = None + self.instances = None + + def __repr__(self): + return 'AutoScalingGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Instances': + self.instances = ResultSet([('member', Instance)]) + return self.instances + elif name == 'LoadBalancerNames': + return self.load_balancers + elif name == 'AvailabilityZones': + return self.availability_zones + elif name == 'EnabledMetrics': + self.metrics = ResultSet([('member', AutoScalingGroupMetric)]) + return self.metrics + else: + return + + def endElement(self, name, value, connection): + if name == 'MinSize': + self.min_size = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'DefaultCooldown': + self.cooldown = value + elif name == 'LaunchConfigurationName': + self.launch_config_name = value + elif name == 'DesiredCapacity': + self.desired_capacity = value + elif name == 'MaxSize': + self.max_size = value + elif name == 'AutoScalingGroupName': + self.name = value + elif name == 'AutoScalingGroupARN': + self.group_arn = value + elif name == 'HealthCheckType': + self.health_check_type = value + elif name == 'HealthCheckGracePeriod': + self.health_check_period = value + elif name == 'SuspendedProcesses': + self.suspended = value + elif name == 'PlacementGroup': + self.placement_group = value + elif name == 'VPCZoneIdentifier': + self.vpc_zone = value + else: + setattr(self, name, value) + + def set_capacity(self, capacity): + """ Set the desired capacity for the group. """ + params = { + 'AutoScalingGroupName' : self.name, + 'DesiredCapacity' : capacity, + } + req = self.connection.get_object('SetDesiredCapacity', params, + Request) + self.connection.last_request = req + return req + + def update(self): + """ Sync local changes with AutoScaling group. """ + return self.connection._update_group('UpdateAutoScalingGroup', self) + + def shutdown_instances(self): + """ Convenience method which shuts down all instances associated with + this group. + """ + self.min_size = 0 + self.max_size = 0 + self.update() + + def get_all_triggers(self): + """ Get all triggers for this auto scaling group. """ + params = {'AutoScalingGroupName' : self.name} + triggers = self.connection.get_list('DescribeTriggers', params, + [('member', Trigger)]) + + # allow triggers to be able to access the autoscale group + for tr in triggers: + tr.autoscale_group = weakref.proxy(self) + + return triggers + + def delete(self): + """ Delete this auto-scaling group. """ + params = {'AutoScalingGroupName' : self.name} + return self.connection.get_object('DeleteAutoScalingGroup', params, + Request) + + def get_activities(self, activity_ids=None, max_records=100): + """ + Get all activies for this group. + """ + return self.connection.get_all_activities(self, activity_ids, max_records) + + +class AutoScalingGroupMetric(object): + def __init__(self, connection=None): + + self.connection = connection + self.metric = None + self.granularity = None + + def __repr__(self): + return 'AutoScalingGroupMetric:%s' % self.metric + + def startElement(self, name, attrs, connection): + return + + def endElement(self, name, value, connection): + if name == 'Metric': + self.metric = value + elif name == 'Granularity': + self.granularity = value + else: + setattr(self, name, value) diff --git a/backup/src/boto/ec2/autoscale/instance.py b/backup/src/boto/ec2/autoscale/instance.py new file mode 100644 index 0000000..ffdd5b1 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/instance.py @@ -0,0 +1,46 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Instance(object): + def __init__(self, connection=None): + self.connection = connection + self.instance_id = '' + self.lifecycle_state = None + self.availability_zone = '' + + def __repr__(self): + return 'Instance:%s' % self.instance_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'InstanceId': + self.instance_id = value + elif name == 'LifecycleState': + self.lifecycle_state = value + elif name == 'AvailabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + + diff --git a/backup/src/boto/ec2/autoscale/launchconfig.py b/backup/src/boto/ec2/autoscale/launchconfig.py new file mode 100644 index 0000000..7587cb6 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/launchconfig.py @@ -0,0 +1,98 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +from boto.ec2.autoscale.request import Request +from boto.ec2.elb.listelement import ListElement + + +class LaunchConfiguration(object): + def __init__(self, connection=None, name=None, image_id=None, + key_name=None, security_groups=None, user_data=None, + instance_type='m1.small', kernel_id=None, + ramdisk_id=None, block_device_mappings=None): + """ + A launch configuration. + + :type name: str + :param name: Name of the launch configuration to create. + + :type image_id: str + :param image_id: Unique ID of the Amazon Machine Image (AMI) which was + assigned during registration. + + :type key_name: str + :param key_name: The name of the EC2 key pair. + + :type security_groups: list + :param security_groups: Names of the security groups with which to + associate the EC2 instances. + + """ + self.connection = connection + self.name = name + self.instance_type = instance_type + self.block_device_mappings = block_device_mappings + self.key_name = key_name + sec_groups = security_groups or [] + self.security_groups = ListElement(sec_groups) + self.image_id = image_id + self.ramdisk_id = ramdisk_id + self.created_time = None + self.kernel_id = kernel_id + self.user_data = user_data + self.created_time = None + + def __repr__(self): + return 'LaunchConfiguration:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'SecurityGroups': + return self.security_groups + else: + return + + def endElement(self, name, value, connection): + if name == 'InstanceType': + self.instance_type = value + elif name == 'LaunchConfigurationName': + self.name = value + elif name == 'KeyName': + self.key_name = value + elif name == 'ImageId': + self.image_id = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'KernelId': + self.kernel_id = value + elif name == 'RamdiskId': + self.ramdisk_id = value + elif name == 'UserData': + self.user_data = value + else: + setattr(self, name, value) + + def delete(self): + """ Delete this launch configuration. """ + params = {'LaunchConfigurationName' : self.name} + return self.connection.get_object('DeleteLaunchConfiguration', params, + Request) + diff --git a/backup/src/boto/ec2/autoscale/request.py b/backup/src/boto/ec2/autoscale/request.py new file mode 100644 index 0000000..c066dff --- /dev/null +++ b/backup/src/boto/ec2/autoscale/request.py @@ -0,0 +1,38 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Request(object): + def __init__(self, connection=None): + self.connection = connection + self.request_id = '' + + def __repr__(self): + return 'Request:%s' % self.request_id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'RequestId': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/ec2/autoscale/trigger.py b/backup/src/boto/ec2/autoscale/trigger.py new file mode 100644 index 0000000..2840e67 --- /dev/null +++ b/backup/src/boto/ec2/autoscale/trigger.py @@ -0,0 +1,134 @@ +# Copyright (c) 2009 Reza Lotun http://reza.lotun.name/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import weakref + +from boto.ec2.autoscale.request import Request + + +class Trigger(object): + """ + An auto scaling trigger. + """ + + def __init__(self, connection=None, name=None, autoscale_group=None, + dimensions=None, measure_name=None, + statistic=None, unit=None, period=60, + lower_threshold=None, + lower_breach_scale_increment=None, + upper_threshold=None, + upper_breach_scale_increment=None, + breach_duration=None): + """ + Initialize an auto-scaling trigger object. + + :type name: str + :param name: The name for this trigger + + :type autoscale_group: str + :param autoscale_group: The name of the AutoScalingGroup that will be + associated with the trigger. The AutoScalingGroup + that will be affected by the trigger when it is + activated. + + :type dimensions: list + :param dimensions: List of tuples, i.e. + ('ImageId', 'i-13lasde') etc. + + :type measure_name: str + :param measure_name: The measure name associated with the metric used by + the trigger to determine when to activate, for + example, CPU, network I/O, or disk I/O. + + :type statistic: str + :param statistic: The particular statistic used by the trigger when + fetching metric statistics to examine. + + :type period: int + :param period: The period associated with the metric statistics in + seconds. Valid Values: 60 or a multiple of 60. + + :type unit: str + :param unit: The unit of measurement. + """ + self.name = name + self.connection = connection + self.dimensions = dimensions + self.breach_duration = breach_duration + self.upper_breach_scale_increment = upper_breach_scale_increment + self.created_time = None + self.upper_threshold = upper_threshold + self.status = None + self.lower_threshold = lower_threshold + self.period = period + self.lower_breach_scale_increment = lower_breach_scale_increment + self.statistic = statistic + self.unit = unit + self.namespace = None + if autoscale_group: + self.autoscale_group = weakref.proxy(autoscale_group) + else: + self.autoscale_group = None + self.measure_name = measure_name + + def __repr__(self): + return 'Trigger:%s' % (self.name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'BreachDuration': + self.breach_duration = value + elif name == 'TriggerName': + self.name = value + elif name == 'Period': + self.period = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'Statistic': + self.statistic = value + elif name == 'Unit': + self.unit = value + elif name == 'Namespace': + self.namespace = value + elif name == 'AutoScalingGroupName': + self.autoscale_group_name = value + elif name == 'MeasureName': + self.measure_name = value + else: + setattr(self, name, value) + + def update(self): + """ Write out differences to trigger. """ + self.connection.create_trigger(self) + + def delete(self): + """ Delete this trigger. """ + params = { + 'TriggerName' : self.name, + 'AutoScalingGroupName' : self.autoscale_group_name, + } + req =self.connection.get_object('DeleteTrigger', params, + Request) + self.connection.last_request = req + return req + diff --git a/backup/src/boto/ec2/blockdevicemapping.py b/backup/src/boto/ec2/blockdevicemapping.py new file mode 100644 index 0000000..efbc38b --- /dev/null +++ b/backup/src/boto/ec2/blockdevicemapping.py @@ -0,0 +1,103 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class BlockDeviceType(object): + + def __init__(self, connection=None): + self.connection = connection + self.ephemeral_name = None + self.no_device = False + self.volume_id = None + self.snapshot_id = None + self.status = None + self.attach_time = None + self.delete_on_termination = False + self.size = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name =='volumeId': + self.volume_id = value + elif name == 'virtualName': + self.ephemeral_name = value + elif name =='NoDevice': + self.no_device = (value == 'true') + elif name =='snapshotId': + self.snapshot_id = value + elif name == 'volumeSize': + self.size = int(value) + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'deleteOnTermination': + if value == 'true': + self.delete_on_termination = True + else: + self.delete_on_termination = False + else: + setattr(self, name, value) + +# for backwards compatibility +EBSBlockDeviceType = BlockDeviceType + +class BlockDeviceMapping(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.current_name = None + self.current_value = None + + def startElement(self, name, attrs, connection): + if name == 'ebs': + self.current_value = BlockDeviceType(self) + return self.current_value + + def endElement(self, name, value, connection): + if name == 'device' or name == 'deviceName': + self.current_name = value + elif name == 'item': + self[self.current_name] = self.current_value + + def build_list_params(self, params, prefix=''): + i = 1 + for dev_name in self: + pre = '%sBlockDeviceMapping.%d' % (prefix, i) + params['%s.DeviceName' % pre] = dev_name + block_dev = self[dev_name] + if block_dev.ephemeral_name: + params['%s.VirtualName' % pre] = block_dev.ephemeral_name + else: + if block_dev.no_device: + params['%s.Ebs.NoDevice' % pre] = 'true' + if block_dev.snapshot_id: + params['%s.Ebs.SnapshotId' % pre] = block_dev.snapshot_id + if block_dev.size: + params['%s.Ebs.VolumeSize' % pre] = block_dev.size + if block_dev.delete_on_termination: + params['%s.Ebs.DeleteOnTermination' % pre] = 'true' + else: + params['%s.Ebs.DeleteOnTermination' % pre] = 'false' + i += 1 diff --git a/backup/src/boto/ec2/bundleinstance.py b/backup/src/boto/ec2/bundleinstance.py new file mode 100644 index 0000000..9651992 --- /dev/null +++ b/backup/src/boto/ec2/bundleinstance.py @@ -0,0 +1,78 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Bundle Task +""" + +from boto.ec2.ec2object import EC2Object + +class BundleInstanceTask(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.instance_id = None + self.progress = None + self.start_time = None + self.state = None + self.bucket = None + self.prefix = None + self.upload_policy = None + self.upload_policy_signature = None + self.update_time = None + self.code = None + self.message = None + + def __repr__(self): + return 'BundleInstanceTask:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'bundleId': + self.id = value + elif name == 'instanceId': + self.instance_id = value + elif name == 'progress': + self.progress = value + elif name == 'startTime': + self.start_time = value + elif name == 'state': + self.state = value + elif name == 'bucket': + self.bucket = value + elif name == 'prefix': + self.prefix = value + elif name == 'uploadPolicy': + self.upload_policy = value + elif name == 'uploadPolicySignature': + self.upload_policy_signature = value + elif name == 'updateTime': + self.update_time = value + elif name == 'code': + self.code = value + elif name == 'message': + self.message = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/ec2/buyreservation.py b/backup/src/boto/ec2/buyreservation.py new file mode 100644 index 0000000..fcd8a77 --- /dev/null +++ b/backup/src/boto/ec2/buyreservation.py @@ -0,0 +1,84 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto.ec2 +from boto.sdb.db.property import StringProperty, IntegerProperty +from boto.manage import propget + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', + 'c1.medium', 'c1.xlarge', 'm2.xlarge', + 'm2.2xlarge', 'm2.4xlarge', 'cc1.4xlarge', + 't1.micro'] + +class BuyReservation(object): + + def get_region(self, params): + if not params.get('region', None): + prop = StringProperty(name='region', verbose_name='EC2 Region', + choices=boto.ec2.regions) + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get(self, params): + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + +if __name__ == "__main__": + obj = BuyReservation() + params = {} + obj.get(params) + offerings = obj.ec2.get_all_reserved_instances_offerings(instance_type=params['instance_type'], + availability_zone=params['zone'].name) + print '\nThe following Reserved Instances Offerings are available:\n' + for offering in offerings: + offering.describe() + prop = StringProperty(name='offering', verbose_name='Offering', + choices=offerings) + offering = propget.get(prop) + print '\nYou have chosen this offering:' + offering.describe() + unit_price = float(offering.fixed_price) + total_price = unit_price * params['quantity'] + print '!!! You are about to purchase %d of these offerings for a total of $%.2f !!!' % (params['quantity'], total_price) + answer = raw_input('Are you sure you want to do this? If so, enter YES: ') + if answer.strip().lower() == 'yes': + offering.purchase(params['quantity']) + else: + print 'Purchase cancelled' diff --git a/backup/src/boto/ec2/cloudwatch/__init__.py b/backup/src/boto/ec2/cloudwatch/__init__.py new file mode 100644 index 0000000..a02baa3 --- /dev/null +++ b/backup/src/boto/ec2/cloudwatch/__init__.py @@ -0,0 +1,502 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +CloudWatch service from AWS. + +The 5 Minute How-To Guide +------------------------- +First, make sure you have something to monitor. You can either create a +LoadBalancer or enable monitoring on an existing EC2 instance. To enable +monitoring, you can either call the monitor_instance method on the +EC2Connection object or call the monitor method on the Instance object. + +It takes a while for the monitoring data to start accumulating but once +it does, you can do this: + +>>> import boto +>>> c = boto.connect_cloudwatch() +>>> metrics = c.list_metrics() +>>> metrics +[Metric:NetworkIn, + Metric:NetworkOut, + Metric:NetworkOut(InstanceType,m1.small), + Metric:NetworkIn(InstanceId,i-e573e68c), + Metric:CPUUtilization(InstanceId,i-e573e68c), + Metric:DiskWriteBytes(InstanceType,m1.small), + Metric:DiskWriteBytes(ImageId,ami-a1ffb63), + Metric:NetworkOut(ImageId,ami-a1ffb63), + Metric:DiskWriteOps(InstanceType,m1.small), + Metric:DiskReadBytes(InstanceType,m1.small), + Metric:DiskReadOps(ImageId,ami-a1ffb63), + Metric:CPUUtilization(InstanceType,m1.small), + Metric:NetworkIn(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceType,m1.small), + Metric:DiskReadBytes, + Metric:CPUUtilization, + Metric:DiskWriteBytes(InstanceId,i-e573e68c), + Metric:DiskWriteOps(InstanceId,i-e573e68c), + Metric:DiskWriteOps, + Metric:DiskReadOps, + Metric:CPUUtilization(ImageId,ami-a1ffb63), + Metric:DiskReadOps(InstanceId,i-e573e68c), + Metric:NetworkOut(InstanceId,i-e573e68c), + Metric:DiskReadBytes(ImageId,ami-a1ffb63), + Metric:DiskReadBytes(InstanceId,i-e573e68c), + Metric:DiskWriteBytes, + Metric:NetworkIn(InstanceType,m1.small), + Metric:DiskWriteOps(ImageId,ami-a1ffb63)] + +The list_metrics call will return a list of all of the available metrics +that you can query against. Each entry in the list is a Metric object. +As you can see from the list above, some of the metrics are generic metrics +and some have Dimensions associated with them (e.g. InstanceType=m1.small). +The Dimension can be used to refine your query. So, for example, I could +query the metric Metric:CPUUtilization which would create the desired statistic +by aggregating cpu utilization data across all sources of information available +or I could refine that by querying the metric +Metric:CPUUtilization(InstanceId,i-e573e68c) which would use only the data +associated with the instance identified by the instance ID i-e573e68c. + +Because for this example, I'm only monitoring a single instance, the set +of metrics available to me are fairly limited. If I was monitoring many +instances, using many different instance types and AMI's and also several +load balancers, the list of available metrics would grow considerably. + +Once you have the list of available metrics, you can actually +query the CloudWatch system for that metric. Let's choose the CPU utilization +metric for our instance. + +>>> m = metrics[5] +>>> m +Metric:CPUUtilization(InstanceId,i-e573e68c) + +The Metric object has a query method that lets us actually perform +the query against the collected data in CloudWatch. To call that, +we need a start time and end time to control the time span of data +that we are interested in. For this example, let's say we want the +data for the previous hour: + +>>> import datetime +>>> end = datetime.datetime.now() +>>> start = end - datetime.timedelta(hours=1) + +We also need to supply the Statistic that we want reported and +the Units to use for the results. The Statistic can be one of these +values: + +['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount'] + +And Units must be one of the following: + +['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', +'Bytes/Second', 'Bits/Second', 'Count/Second'] + +The query method also takes an optional parameter, period. This +parameter controls the granularity (in seconds) of the data returned. +The smallest period is 60 seconds and the value must be a multiple +of 60 seconds. So, let's ask for the average as a percent: + +>>> datapoints = m.query(start, end, 'Average', 'Percent') +>>> len(datapoints) +60 + +Our period was 60 seconds and our duration was one hour so +we should get 60 data points back and we can see that we did. +Each element in the datapoints list is a DataPoint object +which is a simple subclass of a Python dict object. Each +Datapoint object contains all of the information available +about that particular data point. + +>>> d = datapoints[0] +>>> d +{u'Average': 0.0, + u'SampleCount': 1.0, + u'Timestamp': u'2009-05-21T19:55:00Z', + u'Unit': u'Percent'} + +My server obviously isn't very busy right now! +""" +try: + import simplejson as json +except ImportError: + import json +from boto.connection import AWSQueryConnection +from boto.ec2.cloudwatch.metric import Metric +from boto.ec2.cloudwatch.alarm import MetricAlarm, AlarmHistoryItem +from boto.ec2.cloudwatch.datapoint import Datapoint +from boto.regioninfo import RegionInfo +import boto + +RegionData = { + 'us-east-1' : 'monitoring.us-east-1.amazonaws.com', + 'us-west-1' : 'monitoring.us-west-1.amazonaws.com', + 'eu-west-1' : 'monitoring.eu-west-1.amazonaws.com', + 'ap-southeast-1' : 'monitoring.ap-southeast-1.amazonaws.com'} + +def regions(): + """ + Get all available regions for the CloudWatch service. + + :rtype: list + :return: A list of :class:`boto.RegionInfo` instances + """ + regions = [] + for region_name in RegionData: + region = RegionInfo(name=region_name, + endpoint=RegionData[region_name], + connection_cls=CloudWatchConnection) + regions.append(region) + return regions + +def connect_to_region(region_name, **kw_params): + """ + Given a valid region name, return a + :class:`boto.ec2.cloudwatch.CloudWatchConnection`. + + :param str region_name: The name of the region to connect to. + + :rtype: :class:`boto.ec2.CloudWatchConnection` or ``None`` + :return: A connection to the given region, or None if an invalid region + name is given + """ + for region in regions(): + if region.name == region_name: + return region.connect(**kw_params) + return None + + +class CloudWatchConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'cloudwatch_version', '2010-08-01') + DefaultRegionName = boto.config.get('Boto', 'cloudwatch_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'cloudwatch_region_endpoint', + 'monitoring.amazonaws.com') + + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to EC2 Monitoring Service. + + B{Note:} The host argument is overridden by the host specified in the + boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint) + self.region = region + + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['ec2'] + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_metric_statistics(self, period, start_time, end_time, metric_name, + namespace, statistics, dimensions=None, unit=None): + """ + Get time-series data for one or more statistics of a given metric. + + :type metric_name: string + :param metric_name: CPUUtilization|NetworkIO-in|NetworkIO-out|DiskIO-ALL-read| + DiskIO-ALL-write|DiskIO-ALL-read-bytes|DiskIO-ALL-write-bytes + + :rtype: list + """ + params = {'Period' : period, + 'MetricName' : metric_name, + 'Namespace' : namespace, + 'StartTime' : start_time.isoformat(), + 'EndTime' : end_time.isoformat()} + self.build_list_params(params, statistics, 'Statistics.member.%d') + if dimensions: + i = 1 + for name in dimensions: + params['Dimensions.member.%d.Name' % i] = name + params['Dimensions.member.%d.Value' % i] = dimensions[name] + i += 1 + return self.get_list('GetMetricStatistics', params, [('member', Datapoint)]) + + def list_metrics(self, next_token=None): + """ + Returns a list of the valid metrics for which there is recorded data available. + + :type next_token: string + :param next_token: A maximum of 500 metrics will be returned at one time. + If more results are available, the ResultSet returned + will contain a non-Null next_token attribute. Passing + that token as a parameter to list_metrics will retrieve + the next page of metrics. + """ + params = {} + if next_token: + params['NextToken'] = next_token + return self.get_list('ListMetrics', params, [('member', Metric)]) + + def describe_alarms(self, action_prefix=None, alarm_name_prefix=None, alarm_names=None, + max_records=None, state_value=None, next_token=None): + """ + Retrieves alarms with the specified names. If no name is specified, all + alarms for the user are returned. Alarms can be retrieved by using only + a prefix for the alarm name, the alarm state, or a prefix for any + action. + + :type action_prefix: string + :param action_name: The action name prefix. + + :type alarm_name_prefix: string + :param alarm_name_prefix: The alarm name prefix. AlarmNames cannot be specified + if this parameter is specified. + + :type alarm_names: list + :param alarm_names: A list of alarm names to retrieve information for. + + :type max_records: int + :param max_records: The maximum number of alarm descriptions to retrieve. + + :type state_value: string + :param state_value: The state value to be used in matching alarms. + + :type next_token: string + :param next_token: The token returned by a previous call to indicate that there is more data. + + :rtype list + """ + params = {} + if action_prefix: + params['ActionPrefix'] = action_prefix + if alarm_name_prefix: + params['AlarmNamePrefix'] = alarm_name_prefix + elif alarm_names: + self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') + if max_records: + params['MaxRecords'] = max_records + if next_token: + params['NextToken'] = next_token + if state_value: + params['StateValue'] = state_value + return self.get_list('DescribeAlarms', params, [('member', MetricAlarm)]) + + def describe_alarm_history(self, alarm_name=None, start_date=None, end_date=None, + max_records=None, history_item_type=None, next_token=None): + """ + Retrieves history for the specified alarm. Filter alarms by date range + or item type. If an alarm name is not specified, Amazon CloudWatch + returns histories for all of the owner's alarms. + + Amazon CloudWatch retains the history of deleted alarms for a period of + six weeks. If an alarm has been deleted, its history can still be + queried. + + :type alarm_name: string + :param alarm_name: The name of the alarm. + + :type start_date: datetime + :param start_date: The starting date to retrieve alarm history. + + :type end_date: datetime + :param end_date: The starting date to retrieve alarm history. + + :type history_item_type: string + :param history_item_type: The type of alarm histories to retreive (ConfigurationUpdate | StateUpdate | Action) + + :type max_records: int + :param max_records: The maximum number of alarm descriptions to retrieve. + + :type next_token: string + :param next_token: The token returned by a previous call to indicate that there is more data. + + :rtype list + """ + params = {} + if alarm_name: + params['AlarmName'] = alarm_name + if start_date: + params['StartDate'] = start_date.isoformat() + if end_date: + params['EndDate'] = end_date.isoformat() + if history_item_type: + params['HistoryItemType'] = history_item_type + if max_records: + params['MaxRecords'] = max_records + if next_token: + params['NextToken'] = next_token + return self.get_list('DescribeAlarmHistory', params, [('member', AlarmHistoryItem)]) + + def describe_alarms_for_metric(self, metric_name, namespace, period=None, statistic=None, dimensions=None, unit=None): + """ + Retrieves all alarms for a single metric. Specify a statistic, period, + or unit to filter the set of alarms further. + + :type metric_name: string + :param metric_name: The name of the metric + + :type namespace: string + :param namespace: The namespace of the metric. + + :type period: int + :param period: The period in seconds over which the statistic is applied. + + :type statistic: string + :param statistic: The statistic for the metric. + + :type dimensions: list + + :type unit: string + + :rtype list + """ + params = { + 'MetricName' : metric_name, + 'Namespace' : namespace, + } + if period: + params['Period'] = period + if statistic: + params['Statistic'] = statistic + if dimensions: + self.build_list_params(params, dimensions, 'Dimensions.member.%s') + if unit: + params['Unit'] = unit + return self.get_list('DescribeAlarmsForMetric', params, [('member', MetricAlarm)]) + + def put_metric_alarm(self, alarm): + """ + Creates or updates an alarm and associates it with the specified Amazon + CloudWatch metric. Optionally, this operation can associate one or more + Amazon Simple Notification Service resources with the alarm. + + When this operation creates an alarm, the alarm state is immediately + set to INSUFFICIENT_DATA. The alarm is evaluated and its StateValue is + set appropriately. Any actions associated with the StateValue is then + executed. + + When updating an existing alarm, its StateValue is left unchanged. + + :type alarm: boto.ec2.cloudwatch.alarm.MetricAlarm + :param alarm: MetricAlarm object. + """ + params = { + 'AlarmName' : alarm.name, + 'MetricName' : alarm.metric, + 'Namespace' : alarm.namespace, + 'Statistic' : alarm.statistic, + 'ComparisonOperator' : MetricAlarm._cmp_map[alarm.comparison], + 'Threshold' : alarm.threshold, + 'EvaluationPeriods' : alarm.evaluation_periods, + 'Period' : alarm.period, + } + if alarm.actions_enabled is not None: + params['ActionsEnabled'] = alarm.actions_enabled + if alarm.alarm_actions: + self.build_list_params(params, alarm.alarm_actions, 'AlarmActions.member.%s') + if alarm.description: + params['AlarmDescription'] = alarm.description + if alarm.dimensions: + self.build_list_params(params, alarm.dimensions, 'Dimensions.member.%s') + if alarm.insufficient_data_actions: + self.build_list_params(params, alarm.insufficient_data_actions, 'InsufficientDataActions.member.%s') + if alarm.ok_actions: + self.build_list_params(params, alarm.ok_actions, 'OKActions.member.%s') + if alarm.unit: + params['Unit'] = alarm.unit + alarm.connection = self + return self.get_status('PutMetricAlarm', params) + create_alarm = put_metric_alarm + update_alarm = put_metric_alarm + + def delete_alarms(self, alarms): + """ + Deletes all specified alarms. In the event of an error, no alarms are deleted. + + :type alarms: list + :param alarms: List of alarm names. + """ + params = {} + self.build_list_params(params, alarms, 'AlarmNames.member.%s') + return self.get_status('DeleteAlarms', params) + + def set_alarm_state(self, alarm_name, state_reason, state_value, state_reason_data=None): + """ + Temporarily sets the state of an alarm. When the updated StateValue + differs from the previous value, the action configured for the + appropriate state is invoked. This is not a permanent change. The next + periodic alarm check (in about a minute) will set the alarm to its + actual state. + + :type alarm_name: string + :param alarm_name: Descriptive name for alarm. + + :type state_reason: string + :param state_reason: Human readable reason. + + :type state_value: string + :param state_value: OK | ALARM | INSUFFICIENT_DATA + + :type state_reason_data: string + :param state_reason_data: Reason string (will be jsonified). + """ + params = { + 'AlarmName' : alarm_name, + 'StateReason' : state_reason, + 'StateValue' : state_value, + } + if state_reason_data: + params['StateReasonData'] = json.dumps(state_reason_data) + + return self.get_status('SetAlarmState', params) + + def enable_alarm_actions(self, alarm_names): + """ + Enables actions for the specified alarms. + + :type alarms: list + :param alarms: List of alarm names. + """ + params = {} + self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') + return self.get_status('EnableAlarmActions', params) + + def disable_alarm_actions(self, alarm_names): + """ + Disables actions for the specified alarms. + + :type alarms: list + :param alarms: List of alarm names. + """ + params = {} + self.build_list_params(params, alarm_names, 'AlarmNames.member.%s') + return self.get_status('DisableAlarmActions', params) + diff --git a/backup/src/boto/ec2/cloudwatch/alarm.py b/backup/src/boto/ec2/cloudwatch/alarm.py new file mode 100644 index 0000000..a43af22 --- /dev/null +++ b/backup/src/boto/ec2/cloudwatch/alarm.py @@ -0,0 +1,189 @@ +# Copyright (c) 2010 Reza Lotun http://reza.lotun.name +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from datetime import datetime +from boto.resultset import ResultSet +from boto.ec2.cloudwatch.listelement import ListElement +import json + + +class MetricAlarm(object): + + OK = 'OK' + ALARM = 'ALARM' + INSUFFICIENT_DATA = 'INSUFFICIENT_DATA' + + _cmp_map = { + '>=' : 'GreaterThanOrEqualToThreshold', + '>' : 'GreaterThanThreshold', + '<' : 'LessThanThreshold', + '<=' : 'LessThanOrEqualToThreshold', + } + _rev_cmp_map = dict((v, k) for (k, v) in _cmp_map.iteritems()) + + def __init__(self, connection=None, name=None, metric=None, + namespace=None, statistic=None, comparison=None, threshold=None, + period=None, evaluation_periods=None): + """ + Creates a new Alarm. + + :type name: str + :param name: Name of alarm. + + :type metric: str + :param metric: Name of alarm's associated metric. + + :type namespace: str + :param namespace: The namespace for the alarm's metric. + + :type statistic: str + :param statistic: The statistic to apply to the alarm's associated metric. Can + be one of 'SampleCount', 'Average', 'Sum', 'Minimum', 'Maximum' + + :type comparison: str + :param comparison: Comparison used to compare statistic with threshold. Can be + one of '>=', '>', '<', '<=' + + :type threshold: float + :param threshold: The value against which the specified statistic is compared. + + :type period: int + :param period: The period in seconds over which teh specified statistic is applied. + + :type evaluation_periods: int + :param evaluation_period: The number of periods over which data is compared to + the specified threshold + """ + self.name = name + self.connection = connection + self.metric = metric + self.namespace = namespace + self.statistic = statistic + self.threshold = float(threshold) if threshold is not None else None + self.comparison = self._cmp_map.get(comparison) + self.period = int(period) if period is not None else None + self.evaluation_periods = int(evaluation_periods) if evaluation_periods is not None else None + self.actions_enabled = None + self.alarm_arn = None + self.last_updated = None + self.description = '' + self.dimensions = [] + self.insufficient_data_actions = [] + self.ok_actions = [] + self.state_reason = None + self.state_value = None + self.unit = None + alarm_action = [] + self.alarm_actions = ListElement(alarm_action) + + def __repr__(self): + return 'MetricAlarm:%s[%s(%s) %s %s]' % (self.name, self.metric, self.statistic, self.comparison, self.threshold) + + def startElement(self, name, attrs, connection): + if name == 'AlarmActions': + return self.alarm_actions + else: + pass + + def endElement(self, name, value, connection): + if name == 'ActionsEnabled': + self.actions_enabled = value + elif name == 'AlarmArn': + self.alarm_arn = value + elif name == 'AlarmConfigurationUpdatedTimestamp': + self.last_updated = value + elif name == 'AlarmDescription': + self.description = value + elif name == 'AlarmName': + self.name = value + elif name == 'ComparisonOperator': + setattr(self, 'comparison', self._rev_cmp_map[value]) + elif name == 'EvaluationPeriods': + self.evaluation_periods = int(value) + elif name == 'MetricName': + self.metric = value + elif name == 'Namespace': + self.namespace = value + elif name == 'Period': + self.period = int(value) + elif name == 'StateReason': + self.state_reason = value + elif name == 'StateValue': + self.state_value = value + elif name == 'Statistic': + self.statistic = value + elif name == 'Threshold': + self.threshold = float(value) + elif name == 'Unit': + self.unit = value + else: + setattr(self, name, value) + + def set_state(self, value, reason, data=None): + """ Temporarily sets the state of an alarm. + + :type value: str + :param value: OK | ALARM | INSUFFICIENT_DATA + + :type reason: str + :param reason: Reason alarm set (human readable). + + :type data: str + :param data: Reason data (will be jsonified). + """ + return self.connection.set_alarm_state(self.name, reason, value, data) + + def update(self): + return self.connection.update_alarm(self) + + def enable_actions(self): + return self.connection.enable_alarm_actions([self.name]) + + def disable_actions(self): + return self.connection.disable_alarm_actions([self.name]) + + def describe_history(self, start_date=None, end_date=None, max_records=None, history_item_type=None, next_token=None): + return self.connection.describe_alarm_history(self.name, start_date, end_date, + max_records, history_item_type, next_token) + + +class AlarmHistoryItem(object): + def __init__(self, connection=None): + self.connection = connection + + def __repr__(self): + return 'AlarmHistory:%s[%s at %s]' % (self.name, self.summary, self.timestamp) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'AlarmName': + self.name = value + elif name == 'HistoryData': + self.data = json.loads(value) + elif name == 'HistoryItemType': + self.tem_type = value + elif name == 'HistorySummary': + self.summary = value + elif name == 'Timestamp': + self.timestamp = datetime.strptime(value, '%Y-%m-%dT%H:%M:%S.%fZ') diff --git a/backup/src/boto/ec2/cloudwatch/datapoint.py b/backup/src/boto/ec2/cloudwatch/datapoint.py new file mode 100644 index 0000000..d4350ce --- /dev/null +++ b/backup/src/boto/ec2/cloudwatch/datapoint.py @@ -0,0 +1,40 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from datetime import datetime + +class Datapoint(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ['Average', 'Maximum', 'Minimum', 'Sum', 'SampleCount']: + self[name] = float(value) + elif name == 'Timestamp': + self[name] = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ') + elif name != 'member': + self[name] = value + diff --git a/backup/src/boto/ec2/cloudwatch/listelement.py b/backup/src/boto/ec2/cloudwatch/listelement.py new file mode 100644 index 0000000..5be4599 --- /dev/null +++ b/backup/src/boto/ec2/cloudwatch/listelement.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ListElement(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'member': + self.append(value) + + diff --git a/backup/src/boto/ec2/cloudwatch/metric.py b/backup/src/boto/ec2/cloudwatch/metric.py new file mode 100644 index 0000000..cd8c4bc --- /dev/null +++ b/backup/src/boto/ec2/cloudwatch/metric.py @@ -0,0 +1,80 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class Dimensions(dict): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Name': + self._name = value + elif name == 'Value': + self[self._name] = value + elif name != 'Dimensions' and name != 'member': + self[name] = value + +class Metric(object): + + Statistics = ['Minimum', 'Maximum', 'Sum', 'Average', 'SampleCount'] + Units = ['Seconds', 'Percent', 'Bytes', 'Bits', 'Count', + 'Bytes/Second', 'Bits/Second', 'Count/Second'] + + def __init__(self, connection=None): + self.connection = connection + self.name = None + self.namespace = None + self.dimensions = None + + def __repr__(self): + s = 'Metric:%s' % self.name + if self.dimensions: + for name,value in self.dimensions.items(): + s += '(%s,%s)' % (name, value) + return s + + def startElement(self, name, attrs, connection): + if name == 'Dimensions': + self.dimensions = Dimensions() + return self.dimensions + + def endElement(self, name, value, connection): + if name == 'MetricName': + self.name = value + elif name == 'Namespace': + self.namespace = value + else: + setattr(self, name, value) + + def query(self, start_time, end_time, statistic, unit=None, period=60): + return self.connection.get_metric_statistics(period, start_time, end_time, + self.name, self.namespace, [statistic], + self.dimensions, unit) + + def describe_alarms(self, period=None, statistic=None, dimensions=None, unit=None): + return self.connection.describe_alarms_for_metric(self.name, + self.namespace, + period, + statistic, + dimensions, + unit) + diff --git a/backup/src/boto/ec2/connection.py b/backup/src/boto/ec2/connection.py new file mode 100644 index 0000000..6ebbef7 --- /dev/null +++ b/backup/src/boto/ec2/connection.py @@ -0,0 +1,2319 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +import base64 +import warnings +from datetime import datetime +from datetime import timedelta +import boto +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.ec2.image import Image, ImageAttribute +from boto.ec2.instance import Reservation, Instance, ConsoleOutput, InstanceAttribute +from boto.ec2.keypair import KeyPair +from boto.ec2.address import Address +from boto.ec2.volume import Volume +from boto.ec2.snapshot import Snapshot +from boto.ec2.snapshot import SnapshotAttribute +from boto.ec2.zone import Zone +from boto.ec2.securitygroup import SecurityGroup +from boto.ec2.regioninfo import RegionInfo +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.reservedinstance import ReservedInstancesOffering, ReservedInstance +from boto.ec2.spotinstancerequest import SpotInstanceRequest +from boto.ec2.spotpricehistory import SpotPriceHistory +from boto.ec2.spotdatafeedsubscription import SpotDatafeedSubscription +from boto.ec2.bundleinstance import BundleInstanceTask +from boto.ec2.placementgroup import PlacementGroup +from boto.ec2.tag import Tag +from boto.exception import EC2ResponseError + +#boto.set_stream_logger('ec2') + +class EC2Connection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'ec2_version', '2010-08-31') + DefaultRegionName = boto.config.get('Boto', 'ec2_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'ec2_region_endpoint', + 'ec2.amazonaws.com') + ResponseError = EC2ResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, host=None, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to EC2. + + B{Note:} The host argument is overridden by the host specified in the + boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['ec2'] + + def get_params(self): + """ + Returns a dictionary containing the value of of all of the keyword + arguments passed when constructing this connection. + """ + param_names = ['aws_access_key_id', 'aws_secret_access_key', 'is_secure', + 'port', 'proxy', 'proxy_port', 'proxy_user', 'proxy_pass', + 'debug', 'https_connection_factory'] + params = {} + for name in param_names: + params[name] = getattr(self, name) + return params + + def build_filter_params(self, params, filters): + i = 1 + for name in filters: + aws_name = name.replace('_', '-') + params['Filter.%d.Name' % i] = aws_name + value = filters[name] + if not isinstance(value, list): + value = [value] + j = 1 + for v in value: + params['Filter.%d.Value.%d' % (i,j)] = v + j += 1 + i += 1 + + # Image methods + + def get_all_images(self, image_ids=None, owners=None, + executable_by=None, filters=None): + """ + Retrieve all the EC2 images available on your account. + + :type image_ids: list + :param image_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :type executable_by: list + :param executable_by: Returns AMIs for which the specified + user ID has explicit launch permissions + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {} + if image_ids: + self.build_list_params(params, image_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + if executable_by: + self.build_list_params(params, executable_by, 'ExecutableBy') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') + + def get_all_kernels(self, kernel_ids=None, owners=None): + """ + Retrieve all the EC2 kernels available on your account. + Constructs a filter to allow the processing to happen server side. + + :type kernel_ids: list + :param kernel_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {} + if kernel_ids: + self.build_list_params(params, kernel_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + filter = {'image-type' : 'kernel'} + self.build_filter_params(params, filter) + return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') + + def get_all_ramdisks(self, ramdisk_ids=None, owners=None): + """ + Retrieve all the EC2 ramdisks available on your account. + Constructs a filter to allow the processing to happen server side. + + :type ramdisk_ids: list + :param ramdisk_ids: A list of strings with the image IDs wanted + + :type owners: list + :param owners: A list of owner IDs + + :rtype: list + :return: A list of :class:`boto.ec2.image.Image` + """ + params = {} + if ramdisk_ids: + self.build_list_params(params, ramdisk_ids, 'ImageId') + if owners: + self.build_list_params(params, owners, 'Owner') + filter = {'image-type' : 'ramdisk'} + self.build_filter_params(params, filter) + return self.get_list('DescribeImages', params, [('item', Image)], verb='POST') + + def get_image(self, image_id): + """ + Shortcut method to retrieve a specific image (AMI). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.image.Image` + :return: The EC2 Image specified or None if the image is not found + """ + try: + return self.get_all_images(image_ids=[image_id])[0] + except IndexError: # None of those images available + return None + + def register_image(self, name=None, description=None, image_location=None, + architecture=None, kernel_id=None, ramdisk_id=None, + root_device_name=None, block_device_map=None): + """ + Register an image. + + :type name: string + :param name: The name of the AMI. Valid only for EBS-based images. + + :type description: string + :param description: The description of the AMI. + + :type image_location: string + :param image_location: Full path to your AMI manifest in Amazon S3 storage. + Only used for S3-based AMI's. + + :type architecture: string + :param architecture: The architecture of the AMI. Valid choices are: + i386 | x86_64 + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type root_device_name: string + :param root_device_name: The root device name (e.g. /dev/sdh) + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: string + :return: The new image id + """ + params = {} + if name: + params['Name'] = name + if description: + params['Description'] = description + if architecture: + params['Architecture'] = architecture + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if image_location: + params['ImageLocation'] = image_location + if root_device_name: + params['RootDeviceName'] = root_device_name + if block_device_map: + block_device_map.build_list_params(params) + rs = self.get_object('RegisterImage', params, ResultSet, verb='POST') + image_id = getattr(rs, 'imageId', None) + return image_id + + def deregister_image(self, image_id): + """ + Unregister an AMI. + + :type image_id: string + :param image_id: the ID of the Image to unregister + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeregisterImage', {'ImageId':image_id}, verb='POST') + + def create_image(self, instance_id, name, description=None, no_reboot=False): + """ + Will create an AMI from the instance in the running or stopped + state. + + :type instance_id: string + :param instance_id: the ID of the instance to image. + + :type name: string + :param name: The name of the new image + + :type description: string + :param description: An optional human-readable string describing + the contents and purpose of the AMI. + + :type no_reboot: bool + :param no_reboot: An optional flag indicating that the bundling process + should not attempt to shutdown the instance before + bundling. If this flag is True, the responsibility + of maintaining file system integrity is left to the + owner of the instance. + + :rtype: string + :return: The new image id + """ + params = {'InstanceId' : instance_id, + 'Name' : name} + if description: + params['Description'] = description + if no_reboot: + params['NoReboot'] = 'true' + img = self.get_object('CreateImage', params, Image, verb='POST') + return img.id + + # ImageAttribute methods + + def get_image_attribute(self, image_id, attribute='launchPermission'): + """ + Gets an attribute from an image. + + :type image_id: string + :param image_id: The Amazon image id for which you want info about + + :type attribute: string + :param attribute: The attribute you need information about. + Valid choices are: + * launchPermission + * productCodes + * blockDeviceMapping + + :rtype: :class:`boto.ec2.image.ImageAttribute` + :return: An ImageAttribute object representing the value of the + attribute requested + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_object('DescribeImageAttribute', params, ImageAttribute, verb='POST') + + def modify_image_attribute(self, image_id, attribute='launchPermission', + operation='add', user_ids=None, groups=None, + product_codes=None): + """ + Changes an attribute of an image. + + :type image_id: string + :param image_id: The image id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change + + :type operation: string + :param operation: Either add or remove (this is required for changing + launchPermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes + + :type product_codes: list + :param product_codes: Amazon DevPay product code. Currently only one + product code can be associated with an AMI. Once + set, the product code cannot be changed or reset. + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + if product_codes: + self.build_list_params(params, product_codes, 'ProductCode') + return self.get_status('ModifyImageAttribute', params, verb='POST') + + def reset_image_attribute(self, image_id, attribute='launchPermission'): + """ + Resets an attribute of an AMI to its default value. + + :type image_id: string + :param image_id: ID of the AMI for which an attribute will be described + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'ImageId' : image_id, + 'Attribute' : attribute} + return self.get_status('ResetImageAttribute', params, verb='POST') + + # Instance methods + + def get_all_instances(self, instance_ids=None, filters=None): + """ + Retrieve all the instances associated with your account. + + :type instance_ids: list + :param instance_ids: A list of strings of instance IDs + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.instance.Reservation` + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeInstances', params, + [('item', Reservation)], verb='POST') + + def run_instances(self, image_id, min_count=1, max_count=1, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None, + disable_api_termination=False, + instance_initiated_shutdown_behavior=None, + private_ip_address=None, + placement_group=None, client_token=None): + """ + Runs an image on EC2. + + :type image_id: string + :param image_id: The ID of the image to run + + :type min_count: int + :param min_count: The minimum number of instances to launch + + :type max_count: int + :param max_count: The maximum number of instances to launch + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to + associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run: + + * m1.small + * m1.large + * m1.xlarge + * c1.medium + * c1.xlarge + * m2.xlarge + * m2.2xlarge + * m2.4xlarge + * cc1.4xlarge + * t1.micro + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the + instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the + instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances + for VPC. + + :type private_ip_address: string + :param private_ip_address: If you're using VPC, you can optionally use + this parameter to assign the instance a + specific available IP address from the + subnet (e.g., 10.0.0.25). + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :type disable_api_termination: bool + :param disable_api_termination: If True, the instances will be locked + and will not be able to be terminated + via the API. + + :type instance_initiated_shutdown_behavior: string + :param instance_initiated_shutdown_behavior: Specifies whether the + instance's EBS volumes are + stopped (i.e. detached) or + terminated (i.e. deleted) + when the instance is + shutdown by the + owner. Valid values are: + + * stop + * terminate + + :type placement_group: string + :param placement_group: If specified, this is the name of the placement + group in which the instance(s) will be launched. + + :type client_token: string + :param client_token: Unique, case-sensitive identifier you provide + to ensure idempotency of the request. + Maximum 64 ASCII characters + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with + the request for machines + """ + params = {'ImageId':image_id, + 'MinCount':min_count, + 'MaxCount': max_count} + if key_name: + params['KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'SecurityGroup') + if user_data: + params['UserData'] = base64.b64encode(user_data) + if addressing_type: + params['AddressingType'] = addressing_type + if instance_type: + params['InstanceType'] = instance_type + if placement: + params['Placement.AvailabilityZone'] = placement + if placement_group: + params['Placement.GroupName'] = placement_group + if kernel_id: + params['KernelId'] = kernel_id + if ramdisk_id: + params['RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['Monitoring.Enabled'] = 'true' + if subnet_id: + params['SubnetId'] = subnet_id + if private_ip_address: + params['PrivateIpAddress'] = private_ip_address + if block_device_map: + block_device_map.build_list_params(params) + if disable_api_termination: + params['DisableApiTermination'] = 'true' + if instance_initiated_shutdown_behavior: + val = instance_initiated_shutdown_behavior + params['InstanceInitiatedShutdownBehavior'] = val + if client_token: + params['ClientToken'] = client_token + return self.get_object('RunInstances', params, Reservation, verb='POST') + + def terminate_instances(self, instance_ids=None): + """ + Terminate the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('TerminateInstances', params, [('item', Instance)], verb='POST') + + def stop_instances(self, instance_ids=None, force=False): + """ + Stop the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to stop + + :type force: bool + :param force: Forces the instance to stop + + :rtype: list + :return: A list of the instances stopped + """ + params = {} + if force: + params['Force'] = 'true' + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StopInstances', params, [('item', Instance)], verb='POST') + + def start_instances(self, instance_ids=None): + """ + Start the instances specified + + :type instance_ids: list + :param instance_ids: A list of strings of the Instance IDs to start + + :rtype: list + :return: A list of the instances started + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('StartInstances', params, [('item', Instance)], verb='POST') + + def get_console_output(self, instance_id): + """ + Retrieves the console output for the specified instance. + + :type instance_id: string + :param instance_id: The instance ID of a running instance on the cloud. + + :rtype: :class:`boto.ec2.instance.ConsoleOutput` + :return: The console output as a ConsoleOutput object + """ + params = {} + self.build_list_params(params, [instance_id], 'InstanceId') + return self.get_object('GetConsoleOutput', params, ConsoleOutput, verb='POST') + + def reboot_instances(self, instance_ids=None): + """ + Reboot the specified instances. + + :type instance_ids: list + :param instance_ids: The instances to terminate and reboot + """ + params = {} + if instance_ids: + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_status('RebootInstances', params) + + def confirm_product_instance(self, product_code, instance_id): + params = {'ProductCode' : product_code, + 'InstanceId' : instance_id} + rs = self.get_object('ConfirmProductInstance', params, ResultSet, verb='POST') + return (rs.status, rs.ownerId) + + # InstanceAttribute methods + + def get_instance_attribute(self, instance_id, attribute): + """ + Gets an attribute from an instance. + + :type instance_id: string + :param instance_id: The Amazon id of the instance + + :type attribute: string + :param attribute: The attribute you need information about + Valid choices are: + + * instanceType|kernel|ramdisk|userData| + * disableApiTermination| + * instanceInitiatedShutdownBehavior| + * rootDeviceName|blockDeviceMapping + + :rtype: :class:`boto.ec2.image.InstanceAttribute` + :return: An InstanceAttribute object representing the value of the + attribute requested + """ + params = {'InstanceId' : instance_id} + if attribute: + params['Attribute'] = attribute + return self.get_object('DescribeInstanceAttribute', params, + InstanceAttribute, verb='POST') + + def modify_instance_attribute(self, instance_id, attribute, value): + """ + Changes an attribute of an instance + + :type instance_id: string + :param instance_id: The instance id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change. + + * AttributeName - Expected value (default) + * instanceType - A valid instance type (m1.small) + * kernel - Kernel ID (None) + * ramdisk - Ramdisk ID (None) + * userData - Base64 encoded String (None) + * disableApiTermination - Boolean (true) + * instanceInitiatedShutdownBehavior - stop|terminate + * rootDeviceName - device name (None) + + :type value: string + :param value: The new value for the attribute + + :rtype: bool + :return: Whether the operation succeeded or not + """ + # Allow a bool to be passed in for value of disableApiTermination + if attribute == 'disableApiTermination': + if isinstance(value, bool): + if value: + value = 'true' + else: + value = 'false' + params = {'InstanceId' : instance_id, + 'Attribute' : attribute, + 'Value' : value} + return self.get_status('ModifyInstanceAttribute', params, verb='POST') + + def reset_instance_attribute(self, instance_id, attribute): + """ + Resets an attribute of an instance to its default value. + + :type instance_id: string + :param instance_id: ID of the instance + + :type attribute: string + :param attribute: The attribute to reset. Valid values are: + kernel|ramdisk + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'InstanceId' : instance_id, + 'Attribute' : attribute} + return self.get_status('ResetInstanceAttribute', params, verb='POST') + + # Spot Instances + + def get_all_spot_instance_requests(self, request_ids=None, + filters=None): + """ + Retrieve all the spot instances requests associated with your account. + + :type request_ids: list + :param request_ids: A list of strings of spot instance request IDs + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of + :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeSpotInstanceRequests', params, + [('item', SpotInstanceRequest)], verb='POST') + + def get_spot_price_history(self, start_time=None, end_time=None, + instance_type=None, product_description=None): + """ + Retrieve the recent history of spot instances pricing. + + :type start_time: str + :param start_time: An indication of how far back to provide price + changes for. An ISO8601 DateTime string. + + :type end_time: str + :param end_time: An indication of how far forward to provide price + changes for. An ISO8601 DateTime string. + + :type instance_type: str + :param instance_type: Filter responses to a particular instance type. + + :type product_description: str + :param product_descripton: Filter responses to a particular platform. + Valid values are currently: Linux + + :rtype: list + :return: A list tuples containing price and timestamp. + """ + params = {} + if start_time: + params['StartTime'] = start_time + if end_time: + params['EndTime'] = end_time + if instance_type: + params['InstanceType'] = instance_type + if product_description: + params['ProductDescription'] = product_description + return self.get_list('DescribeSpotPriceHistory', params, + [('item', SpotPriceHistory)], verb='POST') + + def request_spot_instances(self, price, image_id, count=1, type='one-time', + valid_from=None, valid_until=None, + launch_group=None, availability_zone_group=None, + key_name=None, security_groups=None, + user_data=None, addressing_type=None, + instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None): + """ + Request instances on the spot market at a particular price. + + :type price: str + :param price: The maximum price of your bid + + :type image_id: string + :param image_id: The ID of the image to run + + :type count: int + :param count: The of instances to requested + + :type type: str + :param type: Type of request. Can be 'one-time' or 'persistent'. + Default is one-time. + + :type valid_from: str + :param valid_from: Start date of the request. An ISO8601 time string. + + :type valid_until: str + :param valid_until: End date of the request. An ISO8601 time string. + + :type launch_group: str + :param launch_group: If supplied, all requests will be fulfilled + as a group. + + :type availability_zone_group: str + :param availability_zone_group: If supplied, all requests will be + fulfilled within a single + availability zone. + + :type key_name: string + :param key_name: The name of the key pair with which to launch instances + + :type security_groups: list of strings + :param security_groups: The names of the security groups with which to + associate instances + + :type user_data: string + :param user_data: The user data passed to the launched instances + + :type instance_type: string + :param instance_type: The type of instance to run: + + * m1.small + * m1.large + * m1.xlarge + * c1.medium + * c1.xlarge + * m2.xlarge + * m2.2xlarge + * m2.4xlarge + * cc1.4xlarge + * t1.micro + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the + instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the + instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances + for VPC. + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :rtype: Reservation + :return: The :class:`boto.ec2.spotinstancerequest.SpotInstanceRequest` + associated with the request for machines + """ + params = {'LaunchSpecification.ImageId':image_id, + 'Type' : type, + 'SpotPrice' : price} + if count: + params['InstanceCount'] = count + if valid_from: + params['ValidFrom'] = valid_from + if valid_until: + params['ValidUntil'] = valid_until + if launch_group: + params['LaunchGroup'] = launch_group + if availability_zone_group: + params['AvailabilityZoneGroup'] = availability_zone_group + if key_name: + params['LaunchSpecification.KeyName'] = key_name + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, SecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, + 'LaunchSpecification.SecurityGroup') + if user_data: + params['LaunchSpecification.UserData'] = base64.b64encode(user_data) + if addressing_type: + params['LaunchSpecification.AddressingType'] = addressing_type + if instance_type: + params['LaunchSpecification.InstanceType'] = instance_type + if placement: + params['LaunchSpecification.Placement.AvailabilityZone'] = placement + if kernel_id: + params['LaunchSpecification.KernelId'] = kernel_id + if ramdisk_id: + params['LaunchSpecification.RamdiskId'] = ramdisk_id + if monitoring_enabled: + params['LaunchSpecification.Monitoring.Enabled'] = 'true' + if subnet_id: + params['LaunchSpecification.SubnetId'] = subnet_id + if block_device_map: + block_device_map.build_list_params(params, 'LaunchSpecification.') + return self.get_list('RequestSpotInstances', params, + [('item', SpotInstanceRequest)], + verb='POST') + + + def cancel_spot_instance_requests(self, request_ids): + """ + Cancel the specified Spot Instance Requests. + + :type request_ids: list + :param request_ids: A list of strings of the Request IDs to terminate + + :rtype: list + :return: A list of the instances terminated + """ + params = {} + if request_ids: + self.build_list_params(params, request_ids, 'SpotInstanceRequestId') + return self.get_list('CancelSpotInstanceRequests', params, + [('item', Instance)], verb='POST') + + def get_spot_datafeed_subscription(self): + """ + Return the current spot instance data feed subscription + associated with this account, if any. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` + :return: The datafeed subscription object or None + """ + return self.get_object('DescribeSpotDatafeedSubscription', + None, SpotDatafeedSubscription, verb='POST') + + def create_spot_datafeed_subscription(self, bucket, prefix): + """ + Create a spot instance datafeed subscription for this account. + + :type bucket: str or unicode + :param bucket: The name of the bucket where spot instance data + will be written. The account issuing this request + must have FULL_CONTROL access to the bucket + specified in the request. + + :type prefix: str or unicode + :param prefix: An optional prefix that will be pre-pended to all + data files written to the bucket. + + :rtype: :class:`boto.ec2.spotdatafeedsubscription.SpotDatafeedSubscription` + :return: The datafeed subscription object or None + """ + params = {'Bucket' : bucket} + if prefix: + params['Prefix'] = prefix + return self.get_object('CreateSpotDatafeedSubscription', + params, SpotDatafeedSubscription, verb='POST') + + def delete_spot_datafeed_subscription(self): + """ + Delete the current spot instance data feed subscription + associated with this account + + :rtype: bool + :return: True if successful + """ + return self.get_status('DeleteSpotDatafeedSubscription', None, verb='POST') + + # Zone methods + + def get_all_zones(self, zones=None, filters=None): + """ + Get all Availability Zones associated with the current region. + + :type zones: list + :param zones: Optional list of zones. If this list is present, + only the Zones associated with these zone names + will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list of :class:`boto.ec2.zone.Zone` + :return: The requested Zone objects + """ + params = {} + if zones: + self.build_list_params(params, zones, 'ZoneName') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeAvailabilityZones', params, [('item', Zone)], verb='POST') + + # Address methods + + def get_all_addresses(self, addresses=None, filters=None): + """ + Get all EIP's associated with the current credentials. + + :type addresses: list + :param addresses: Optional list of addresses. If this list is present, + only the Addresses associated with these addresses + will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list of :class:`boto.ec2.address.Address` + :return: The requested Address objects + """ + params = {} + if addresses: + self.build_list_params(params, addresses, 'PublicIp') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeAddresses', params, [('item', Address)], verb='POST') + + def allocate_address(self): + """ + Allocate a new Elastic IP address and associate it with your account. + + :rtype: :class:`boto.ec2.address.Address` + :return: The newly allocated Address + """ + return self.get_object('AllocateAddress', {}, Address, verb='POST') + + def associate_address(self, instance_id, public_ip): + """ + Associate an Elastic IP address with a currently running instance. + + :type instance_id: string + :param instance_id: The ID of the instance + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, 'PublicIp' : public_ip} + return self.get_status('AssociateAddress', params, verb='POST') + + def disassociate_address(self, public_ip): + """ + Disassociate an Elastic IP address from a currently running instance. + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('DisassociateAddress', params, verb='POST') + + def release_address(self, public_ip): + """ + Free up an Elastic IP address + + :type public_ip: string + :param public_ip: The public IP address + + :rtype: bool + :return: True if successful + """ + params = {'PublicIp' : public_ip} + return self.get_status('ReleaseAddress', params, verb='POST') + + # Volume methods + + def get_all_volumes(self, volume_ids=None, filters=None): + """ + Get all Volumes associated with the current credentials. + + :type volume_ids: list + :param volume_ids: Optional list of volume ids. If this list is present, + only the volumes associated with these volume ids + will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list of :class:`boto.ec2.volume.Volume` + :return: The requested Volume objects + """ + params = {} + if volume_ids: + self.build_list_params(params, volume_ids, 'VolumeId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeVolumes', params, [('item', Volume)], verb='POST') + + def create_volume(self, size, zone, snapshot=None): + """ + Create a new EBS Volume. + + :type size: int + :param size: The size of the new volume, in GiB + + :type zone: string or :class:`boto.ec2.zone.Zone` + :param zone: The availability zone in which the Volume will be created. + + :type snapshot: string or :class:`boto.ec2.snapshot.Snapshot` + :param snapshot: The snapshot from which the new Volume will be created. + """ + if isinstance(zone, Zone): + zone = zone.name + params = {'AvailabilityZone' : zone} + if size: + params['Size'] = size + if snapshot: + if isinstance(snapshot, Snapshot): + snapshot = snapshot.id + params['SnapshotId'] = snapshot + return self.get_object('CreateVolume', params, Volume, verb='POST') + + def delete_volume(self, volume_id): + """ + Delete an EBS volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be delete. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId': volume_id} + return self.get_status('DeleteVolume', params, verb='POST') + + def attach_volume(self, volume_id, instance_id, device): + """ + Attach an EBS volume to an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + params = {'InstanceId' : instance_id, + 'VolumeId' : volume_id, + 'Device' : device} + return self.get_status('AttachVolume', params, verb='POST') + + def detach_volume(self, volume_id, instance_id=None, + device=None, force=False): + """ + Detach an EBS volume from an EC2 instance. + + :type volume_id: str + :param volume_id: The ID of the EBS volume to be attached. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance from which it will + be detached. + + :type device: str + :param device: The device on the instance through which the + volume is exposted (e.g. /dev/sdh) + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if instance_id: + params['InstanceId'] = instance_id + if device: + params['Device'] = device + if force: + params['Force'] = 'true' + return self.get_status('DetachVolume', params, verb='POST') + + # Snapshot methods + + def get_all_snapshots(self, snapshot_ids=None, + owner=None, restorable_by=None, + filters=None): + """ + Get all EBS Snapshots associated with the current credentials. + + :type snapshot_ids: list + :param snapshot_ids: Optional list of snapshot ids. If this list is + present, only the Snapshots associated with + these snapshot ids will be returned. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + + * self + * amazon + * AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list of :class:`boto.ec2.snapshot.Snapshot` + :return: The requested Snapshot objects + """ + params = {} + if snapshot_ids: + self.build_list_params(params, snapshot_ids, 'SnapshotId') + if owner: + params['Owner'] = owner + if restorable_by: + params['RestorableBy'] = restorable_by + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeSnapshots', params, [('item', Snapshot)], verb='POST') + + def create_snapshot(self, volume_id, description=None): + """ + Create a snapshot of an existing EBS Volume. + + :type volume_id: str + :param volume_id: The ID of the volume to be snapshot'ed + + :type description: str + :param description: A description of the snapshot. + Limited to 255 characters. + + :rtype: bool + :return: True if successful + """ + params = {'VolumeId' : volume_id} + if description: + params['Description'] = description[0:255] + snapshot = self.get_object('CreateSnapshot', params, Snapshot, verb='POST') + volume = self.get_all_volumes([volume_id])[0] + volume_name = volume.tags.get('Name') + if volume_name: + snapshot.add_tag('Name', volume_name) + return snapshot + + def delete_snapshot(self, snapshot_id): + params = {'SnapshotId': snapshot_id} + return self.get_status('DeleteSnapshot', params, verb='POST') + + def trim_snapshots(self, hourly_backups = 8, daily_backups = 7, weekly_backups = 4): + """ + Trim excess snapshots, based on when they were taken. More current snapshots are + retained, with the number retained decreasing as you move back in time. + + If ebs volumes have a 'Name' tag with a value, their snapshots will be assigned the same + tag when they are created. The values of the 'Name' tags for snapshots are used by this + function to group snapshots taken from the same volume (or from a series of like-named + volumes over time) for trimming. + + For every group of like-named snapshots, this function retains the newest and oldest + snapshots, as well as, by default, the first snapshots taken in each of the last eight + hours, the first snapshots taken in each of the last seven days, the first snapshots + taken in the last 4 weeks (counting Midnight Sunday morning as the start of the week), + and the first snapshot from the first Sunday of each month forever. + + :type hourly_backups: int + :param hourly_backups: How many recent hourly backups should be saved. + + :type daily_backups: int + :param daily_backups: How many recent daily backups should be saved. + + :type weekly_backups: int + :param weekly_backups: How many recent weekly backups should be saved. + """ + + # This function first builds up an ordered list of target times that snapshots should be saved for + # (last 8 hours, last 7 days, etc.). Then a map of snapshots is constructed, with the keys being + # the snapshot / volume names and the values being arrays of chornologically sorted snapshots. + # Finally, for each array in the map, we go through the snapshot array and the target time array + # in an interleaved fashion, deleting snapshots whose start_times don't immediately follow a + # target time (we delete a snapshot if there's another snapshot that was made closer to the + # preceding target time). + + now = datetime.utcnow() # work with UTC time, which is what the snapshot start time is reported in + last_hour = datetime(now.year, now.month, now.day, now.hour) + last_midnight = datetime(now.year, now.month, now.day) + last_sunday = datetime(now.year, now.month, now.day) - timedelta(days = (now.weekday() + 1) % 7) + start_of_month = datetime(now.year, now.month, 1) + + target_backup_times = [] + + oldest_snapshot_date = datetime(2007, 1, 1) # there are no snapshots older than 1/1/2007 + + for hour in range(0, hourly_backups): + target_backup_times.append(last_hour - timedelta(hours = hour)) + + for day in range(0, daily_backups): + target_backup_times.append(last_midnight - timedelta(days = day)) + + for week in range(0, weekly_backups): + target_backup_times.append(last_sunday - timedelta(weeks = week)) + + one_day = timedelta(days = 1) + while start_of_month > oldest_snapshot_date: + # append the start of the month to the list of snapshot dates to save: + target_backup_times.append(start_of_month) + # there's no timedelta setting for one month, so instead: + # decrement the day by one, so we go to the final day of the previous month... + start_of_month -= one_day + # ... and then go to the first day of that previous month: + start_of_month = datetime(start_of_month.year, start_of_month.month, 1) + + temp = [] + + for t in target_backup_times: + if temp.__contains__(t) == False: + temp.append(t) + + target_backup_times = temp + target_backup_times.reverse() # make the oldest date first + + # get all the snapshots, sort them by date and time, and organize them into one array for each volume: + all_snapshots = self.get_all_snapshots(owner = 'self') + all_snapshots.sort(cmp = lambda x, y: cmp(x.start_time, y.start_time)) # oldest first + snaps_for_each_volume = {} + for snap in all_snapshots: + # the snapshot name and the volume name are the same. The snapshot name is set from the volume + # name at the time the snapshot is taken + volume_name = snap.tags.get('Name') + if volume_name: + # only examine snapshots that have a volume name + snaps_for_volume = snaps_for_each_volume.get(volume_name) + if not snaps_for_volume: + snaps_for_volume = [] + snaps_for_each_volume[volume_name] = snaps_for_volume + snaps_for_volume.append(snap) + + # Do a running comparison of snapshot dates to desired time periods, keeping the oldest snapshot in each + # time period and deleting the rest: + for volume_name in snaps_for_each_volume: + snaps = snaps_for_each_volume[volume_name] + snaps = snaps[:-1] # never delete the newest snapshot, so remove it from consideration + time_period_number = 0 + snap_found_for_this_time_period = False + for snap in snaps: + check_this_snap = True + while check_this_snap and time_period_number < target_backup_times.__len__(): + snap_date = datetime.strptime(snap.start_time, '%Y-%m-%dT%H:%M:%S.000Z') + if snap_date < target_backup_times[time_period_number]: + # the snap date is before the cutoff date. Figure out if it's the first snap in this + # date range and act accordingly (since both date the date ranges and the snapshots + # are sorted chronologically, we know this snapshot isn't in an earlier date range): + if snap_found_for_this_time_period == True: + if not snap.tags.get('preserve_snapshot'): + # as long as the snapshot wasn't marked with the 'preserve_snapshot' tag, delete it: + self.delete_snapshot(snap.id) + boto.log.info('Trimmed snapshot %s (%s)' % (snap.tags['Name'], snap.start_time)) + # go on and look at the next snapshot, leaving the time period alone + else: + # this was the first snapshot found for this time period. Leave it alone and look at the + # next snapshot: + snap_found_for_this_time_period = True + check_this_snap = False + else: + # the snap is after the cutoff date. Check it against the next cutoff date + time_period_number += 1 + snap_found_for_this_time_period = False + + + def get_snapshot_attribute(self, snapshot_id, + attribute='createVolumePermission'): + """ + Get information about an attribute of a snapshot. Only one attribute + can be specified per call. + + :type snapshot_id: str + :param snapshot_id: The ID of the snapshot. + + :type attribute: str + :param attribute: The requested attribute. Valid values are: + + * createVolumePermission + + :rtype: list of :class:`boto.ec2.snapshotattribute.SnapshotAttribute` + :return: The requested Snapshot attribute + """ + params = {'Attribute' : attribute} + if snapshot_id: + params['SnapshotId'] = snapshot_id + return self.get_object('DescribeSnapshotAttribute', params, + SnapshotAttribute, verb='POST') + + def modify_snapshot_attribute(self, snapshot_id, + attribute='createVolumePermission', + operation='add', user_ids=None, groups=None): + """ + Changes an attribute of an image. + + :type snapshot_id: string + :param snapshot_id: The snapshot id you wish to change + + :type attribute: string + :param attribute: The attribute you wish to change. Valid values are: + createVolumePermission + + :type operation: string + :param operation: Either add or remove (this is required for changing + snapshot ermissions) + + :type user_ids: list + :param user_ids: The Amazon IDs of users to add/remove attributes + + :type groups: list + :param groups: The groups to add/remove attributes. The only valid + value at this time is 'all'. + + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute, + 'OperationType' : operation} + if user_ids: + self.build_list_params(params, user_ids, 'UserId') + if groups: + self.build_list_params(params, groups, 'UserGroup') + return self.get_status('ModifySnapshotAttribute', params, verb='POST') + + def reset_snapshot_attribute(self, snapshot_id, + attribute='createVolumePermission'): + """ + Resets an attribute of a snapshot to its default value. + + :type snapshot_id: string + :param snapshot_id: ID of the snapshot + + :type attribute: string + :param attribute: The attribute to reset + + :rtype: bool + :return: Whether the operation succeeded or not + """ + params = {'SnapshotId' : snapshot_id, + 'Attribute' : attribute} + return self.get_status('ResetSnapshotAttribute', params, verb='POST') + + # Keypair methods + + def get_all_key_pairs(self, keynames=None, filters=None): + """ + Get all key pairs associated with your account. + + :type keynames: list + :param keynames: A list of the names of keypairs to retrieve. + If not provided, all key pairs will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.keypair.KeyPair` + """ + params = {} + if keynames: + self.build_list_params(params, keynames, 'KeyName') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeKeyPairs', params, [('item', KeyPair)], verb='POST') + + def get_key_pair(self, keyname): + """ + Convenience method to retrieve a specific keypair (KeyPair). + + :type image_id: string + :param image_id: the ID of the Image to retrieve + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The KeyPair specified or None if it is not found + """ + try: + return self.get_all_key_pairs(keynames=[keyname])[0] + except IndexError: # None of those key pairs available + return None + + def create_key_pair(self, key_name): + """ + Create a new key pair for your account. + This will create the key pair within the region you + are currently connected to. + + :type key_name: string + :param key_name: The name of the new keypair + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + The material attribute of the new KeyPair object + will contain the the unencrypted PEM encoded RSA private key. + """ + params = {'KeyName':key_name} + return self.get_object('CreateKeyPair', params, KeyPair, verb='POST') + + def delete_key_pair(self, key_name): + """ + Delete a key pair from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'KeyName':key_name} + return self.get_status('DeleteKeyPair', params, verb='POST') + + def import_key_pair(self, key_name, public_key_material): + """ + mports the public key from an RSA key pair that you created + with a third-party tool. + + Supported formats: + + * OpenSSH public key format (e.g., the format + in ~/.ssh/authorized_keys) + + * Base64 encoded DER format + + * SSH public key file format as specified in RFC4716 + + DSA keys are not supported. Make sure your key generator is + set up to create RSA keys. + + Supported lengths: 1024, 2048, and 4096. + + :type key_name: string + :param key_name: The name of the new keypair + + :type public_key_material: string + :param public_key_material: The public key. You must base64 encode + the public key material before sending + it to AWS. + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + The material attribute of the new KeyPair object + will contain the the unencrypted PEM encoded RSA private key. + """ + params = {'KeyName' : key_name, + 'PublicKeyMaterial' : public_key_material} + return self.get_object('ImportKeyPair', params, KeyPair, verb='POST') + + # SecurityGroup methods + + def get_all_security_groups(self, groupnames=None, filters=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be + returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.securitygroup.SecurityGroup` + """ + params = {} + if groupnames: + self.build_list_params(params, groupnames, 'GroupName') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeSecurityGroups', params, + [('item', SecurityGroup)], verb='POST') + + def create_security_group(self, name, description): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + """ + params = {'GroupName':name, 'GroupDescription':description} + group = self.get_object('CreateSecurityGroup', params, SecurityGroup, verb='POST') + group.name = name + group.description = description + return group + + def delete_security_group(self, name): + """ + Delete a security group from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'GroupName':name} + return self.get_status('DeleteSecurityGroup', params, verb='POST') + + def _authorize_deprecated(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None): + """ + This method is called only when someone tries to authorize a group + without specifying a from_port or to_port. Until recently, that was + the only way to do group authorization but the EC2 API has been + changed to now require a from_port and to_port when specifying a + group. This is a much better approach but I don't want to break + existing boto applications that depend on the old behavior, hence + this kludge. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + granting access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security + group you are granting access to. + + :rtype: bool + :return: True if successful. + """ + warnings.warn('FromPort and ToPort now required for group authorization', + DeprecationWarning) + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + return self.get_status('AuthorizeSecurityGroupIngress', params, verb='POST') + + def authorize_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + granting access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security + group you are granting access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + if src_security_group_name: + if from_port is None and to_port is None and ip_protocol is None: + return self._authorize_deprecated(group_name, + src_security_group_name, + src_security_group_owner_id) + params = {'GroupName':group_name} + if src_security_group_name: + params['IpPermissions.1.Groups.1.GroupName'] = src_security_group_name + if src_security_group_owner_id: + params['IpPermissions.1.Groups.1.UserId'] = src_security_group_owner_id + if ip_protocol: + params['IpPermissions.1.IpProtocol'] = ip_protocol + if from_port: + params['IpPermissions.1.FromPort'] = from_port + if to_port: + params['IpPermissions.1.ToPort'] = to_port + if cidr_ip: + params['IpPermissions.1.IpRanges.1.CidrIp'] = cidr_ip + return self.get_status('AuthorizeSecurityGroupIngress', params, verb='POST') + + def _revoke_deprecated(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None): + """ + This method is called only when someone tries to revoke a group + without specifying a from_port or to_port. Until recently, that was + the only way to do group revocation but the EC2 API has been + changed to now require a from_port and to_port when specifying a + group. This is a much better approach but I don't want to break + existing boto applications that depend on the old behavior, hence + this kludge. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + granting access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security + group you are granting access to. + + :rtype: bool + :return: True if successful. + """ + warnings.warn('FromPort and ToPort now required for group authorization', + DeprecationWarning) + params = {'GroupName':group_name} + if src_security_group_name: + params['SourceSecurityGroupName'] = src_security_group_name + if src_security_group_owner_id: + params['SourceSecurityGroupOwnerId'] = src_security_group_owner_id + return self.get_status('RevokeSecurityGroupIngress', params, verb='POST') + + def revoke_security_group(self, group_name, src_security_group_name=None, + src_security_group_owner_id=None, + ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are revoking another + group or you are revoking some ip-based rule. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type src_security_group_name: string + :param src_security_group_name: The name of the security group you are + revoking access to. + + :type src_security_group_owner_id: string + :param src_security_group_owner_id: The ID of the owner of the security + group you are revoking access to. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are disabling + + :type to_port: int + :param to_port: The ending port number you are disabling + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are revoking access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + if src_security_group_name: + if from_port is None and to_port is None and ip_protocol is None: + return self._revoke_deprecated(group_name, + src_security_group_name, + src_security_group_owner_id) + params = {'GroupName':group_name} + if src_security_group_name: + params['IpPermissions.1.Groups.1.GroupName'] = src_security_group_name + if src_security_group_owner_id: + params['IpPermissions.1.Groups.1.UserId'] = src_security_group_owner_id + if ip_protocol: + params['IpPermissions.1.IpProtocol'] = ip_protocol + if from_port: + params['IpPermissions.1.FromPort'] = from_port + if to_port: + params['IpPermissions.1.ToPort'] = to_port + if cidr_ip: + params['IpPermissions.1.IpRanges.1.CidrIp'] = cidr_ip + return self.get_status('RevokeSecurityGroupIngress', params, verb='POST') + + # + # Regions + # + + def get_all_regions(self, region_names=None, filters=None): + """ + Get all available regions for the EC2 service. + + :type region_names: list of str + :param region_names: Names of regions to limit output + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + params = {} + if region_names: + self.build_list_params(params, region_names, 'RegionName') + if filters: + self.build_filter_params(params, filters) + regions = self.get_list('DescribeRegions', params, [('item', RegionInfo)], verb='POST') + for region in regions: + region.connection_cls = EC2Connection + return regions + + # + # Reservation methods + # + + def get_all_reserved_instances_offerings(self, reserved_instances_id=None, + instance_type=None, + availability_zone=None, + product_description=None, + filters=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instances_id: str + :param reserved_instances_id: Displays Reserved Instances with the + specified offering IDs. + + :type instance_type: str + :param instance_type: Displays Reserved Instances of the specified + instance type. + + :type availability_zone: str + :param availability_zone: Displays Reserved Instances within the + specified Availability Zone. + + :type product_description: str + :param product_description: Displays Reserved Instances with the + specified product description. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstancesOffering` + """ + params = {} + if reserved_instances_id: + params['ReservedInstancesId'] = reserved_instances_id + if instance_type: + params['InstanceType'] = instance_type + if availability_zone: + params['AvailabilityZone'] = availability_zone + if product_description: + params['ProductDescription'] = product_description + if filters: + self.build_filter_params(params, filters) + + return self.get_list('DescribeReservedInstancesOfferings', + params, [('item', ReservedInstancesOffering)], verb='POST') + + def get_all_reserved_instances(self, reserved_instances_id=None, + filters=None): + """ + Describes Reserved Instance offerings that are available for purchase. + + :type reserved_instance_ids: list + :param reserved_instance_ids: A list of the reserved instance ids that + will be returned. If not provided, all + reserved instances will be returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.reservedinstance.ReservedInstance` + """ + params = {} + if reserved_instances_id: + self.build_list_params(params, reserved_instances_id, + 'ReservedInstancesId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeReservedInstances', + params, [('item', ReservedInstance)], verb='POST') + + def purchase_reserved_instance_offering(self, reserved_instances_offering_id, + instance_count=1): + """ + Purchase a Reserved Instance for use with your account. + ** CAUTION ** + This request can result in large amounts of money being charged to your + AWS account. Use with caution! + + :type reserved_instances_offering_id: string + :param reserved_instances_offering_id: The offering ID of the Reserved + Instance to purchase + + :type instance_count: int + :param instance_count: The number of Reserved Instances to purchase. + Default value is 1. + + :rtype: :class:`boto.ec2.reservedinstance.ReservedInstance` + :return: The newly created Reserved Instance + """ + params = {'ReservedInstancesOfferingId' : reserved_instances_offering_id, + 'InstanceCount' : instance_count} + return self.get_object('PurchaseReservedInstancesOffering', params, + ReservedInstance, verb='POST') + + # + # Monitoring + # + + def monitor_instances(self, instance_ids): + """ + Enable CloudWatch monitoring for the supplied instances. + + :type instance_id: list of strings + :param instance_id: The instance ids + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {} + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('MonitorInstances', params, + [('item', InstanceInfo)], verb='POST') + + def monitor_instance(self, instance_id): + """ + Deprecated Version, maintained for backward compatibility. + Enable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + return self.monitor_instances([instance_id]) + + def unmonitor_instance(self, instance_ids): + """ + Disable CloudWatch monitoring for the supplied instance. + + :type instance_id: list of string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + params = {} + self.build_list_params(params, instance_ids, 'InstanceId') + return self.get_list('UnmonitorInstances', params, + [('item', InstanceInfo)], verb='POST') + + def unmonitor_instance(self, instance_id): + """ + Deprecated Version, maintained for backward compatibility. + Disable CloudWatch monitoring for the supplied instance. + + :type instance_id: string + :param instance_id: The instance id + + :rtype: list + :return: A list of :class:`boto.ec2.instanceinfo.InstanceInfo` + """ + return self.unmonitor_instances([instance_id]) + + # + # Bundle Windows Instances + # + + def bundle_instance(self, instance_id, + s3_bucket, + s3_prefix, + s3_upload_policy): + """ + Bundle Windows instance. + + :type instance_id: string + :param instance_id: The instance id + + :type s3_bucket: string + :param s3_bucket: The bucket in which the AMI should be stored. + + :type s3_prefix: string + :param s3_prefix: The beginning of the file name for the AMI. + + :type s3_upload_policy: string + :param s3_upload_policy: Base64 encoded policy that specifies condition + and permissions for Amazon EC2 to upload the + user's image into Amazon S3. + """ + + params = {'InstanceId' : instance_id, + 'Storage.S3.Bucket' : s3_bucket, + 'Storage.S3.Prefix' : s3_prefix, + 'Storage.S3.UploadPolicy' : s3_upload_policy} + s3auth = boto.auth.get_auth_handler(None, boto.config, + self.provider, ['s3']) + params['Storage.S3.AWSAccessKeyId'] = self.aws_access_key_id + signature = s3auth.sign_string(s3_upload_policy) + params['Storage.S3.UploadPolicySignature'] = signature + return self.get_object('BundleInstance', params, + BundleInstanceTask, verb='POST') + + def get_all_bundle_tasks(self, bundle_ids=None, filters=None): + """ + Retrieve current bundling tasks. If no bundle id is specified, all + tasks are retrieved. + + :type bundle_ids: list + :param bundle_ids: A list of strings containing identifiers for + previously created bundling tasks. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + """ + + params = {} + if bundle_ids: + self.build_list_params(params, bundle_ids, 'BundleId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeBundleTasks', params, + [('item', BundleInstanceTask)], verb='POST') + + def cancel_bundle_task(self, bundle_id): + """ + Cancel a previously submitted bundle task + + :type bundle_id: string + :param bundle_id: The identifier of the bundle task to cancel. + """ + + params = {'BundleId' : bundle_id} + return self.get_object('CancelBundleTask', params, + BundleInstanceTask, verb='POST') + + def get_password_data(self, instance_id): + """ + Get encrypted administrator password for a Windows instance. + + :type instance_id: string + :param instance_id: The identifier of the instance to retrieve the + password for. + """ + + params = {'InstanceId' : instance_id} + rs = self.get_object('GetPasswordData', params, ResultSet, verb='POST') + return rs.passwordData + + # + # Cluster Placement Groups + # + + def get_all_placement_groups(self, groupnames=None, filters=None): + """ + Get all placement groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of placement groups to retrieve. + If not provided, all placement groups will be + returned. + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: list + :return: A list of :class:`boto.ec2.placementgroup.PlacementGroup` + """ + params = {} + if groupnames: + self.build_list_params(params, groupnames, 'GroupName') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribePlacementGroups', params, + [('item', PlacementGroup)], verb='POST') + + def create_placement_group(self, name, strategy='cluster'): + """ + Create a new placement group for your account. + This will create the placement group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new placement group + + :type strategy: string + :param strategy: The placement strategy of the new placement group. + Currently, the only acceptable value is "cluster". + + :rtype: :class:`boto.ec2.placementgroup.PlacementGroup` + :return: The newly created :class:`boto.ec2.keypair.KeyPair`. + """ + params = {'GroupName':name, 'Strategy':strategy} + group = self.get_status('CreatePlacementGroup', params, verb='POST') + return group + + def delete_placement_group(self, name): + """ + Delete a placement group from your account. + + :type key_name: string + :param key_name: The name of the keypair to delete + """ + params = {'GroupName':name} + return self.get_status('DeletePlacementGroup', params, verb='POST') + + # Tag methods + + def build_tag_param_list(self, params, tags): + keys = tags.keys() + keys.sort() + i = 1 + for key in keys: + value = tags[key] + params['Tag.%d.Key'%i] = key + if value is None: + value = '' + params['Tag.%d.Value'%i] = value + i += 1 + + def get_all_tags(self, tags=None, filters=None): + """ + Retrieve all the metadata tags associated with your account. + + :type tags: list + :param tags: A list of mumble + + :type filters: dict + :param filters: Optional filters that can be used to limit + the results returned. Filters are provided + in the form of a dictionary consisting of + filter names as the key and filter values + as the value. The set of allowable filter + names/values is dependent on the request + being performed. Check the EC2 API guide + for details. + + :rtype: dict + :return: A dictionary containing metadata tags + """ + params = {} + if tags: + self.build_list_params(params, instance_ids, 'InstanceId') + if filters: + self.build_filter_params(params, filters) + return self.get_list('DescribeTags', params, [('item', Tag)], verb='POST') + + def create_tags(self, resource_ids, tags): + """ + Create new metadata tags for the specified resource ids. + + :type resource_ids: list + :param resource_ids: List of strings + + :type tags: dict + :param tags: A dictionary containing the name/value pairs + + """ + params = {} + self.build_list_params(params, resource_ids, 'ResourceId') + self.build_tag_param_list(params, tags) + return self.get_status('CreateTags', params, verb='POST') + + def delete_tags(self, resource_ids, tags): + """ + Delete metadata tags for the specified resource ids. + + :type resource_ids: list + :param resource_ids: List of strings + + :type tags: dict or list + :param tags: Either a dictionary containing name/value pairs + or a list containing just tag names. + If you pass in a dictionary, the values must + match the actual tag values or the tag will + not be deleted. + + """ + if isinstance(tags, list): + tags = {}.fromkeys(tags, None) + params = {} + self.build_list_params(params, resource_ids, 'ResourceId') + self.build_tag_param_list(params, tags) + return self.get_status('DeleteTags', params, verb='POST') + diff --git a/backup/src/boto/ec2/ec2object.py b/backup/src/boto/ec2/ec2object.py new file mode 100644 index 0000000..6e37596 --- /dev/null +++ b/backup/src/boto/ec2/ec2object.py @@ -0,0 +1,102 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Object +""" +from boto.ec2.tag import TagSet + +class EC2Object(object): + + def __init__(self, connection=None): + self.connection = connection + if self.connection and hasattr(self.connection, 'region'): + self.region = connection.region + else: + self.region = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + + +class TaggedEC2Object(EC2Object): + """ + Any EC2 resource that can be tagged should be represented + by a Python object that subclasses this class. This class + has the mechanism in place to handle the tagSet element in + the Describe* responses. If tags are found, it will create + a TagSet object and allow it to parse and collect the tags + into a dict that is stored in the "tags" attribute of the + object. + """ + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.tags = TagSet() + + def startElement(self, name, attrs, connection): + if name == 'tagSet': + return self.tags + else: + return None + + def add_tag(self, key, value=None): + """ + Add a tag to this object. Tag's are stored by AWS and can be used + to organize and filter resources. Adding a tag involves a round-trip + to the EC2 service. + + :type key: str + :param key: The key or name of the tag being stored. + + :type value: str + :param value: An optional value that can be stored with the tag. + """ + status = self.connection.create_tags([self.id], {key : value}) + if self.tags is None: + self.tags = TagSet() + self.tags[key] = value + + def remove_tag(self, key, value=None): + """ + Remove a tag from this object. Removing a tag involves a round-trip + to the EC2 service. + + :type key: str + :param key: The key or name of the tag being stored. + + :type value: str + :param value: An optional value that can be stored with the tag. + If a value is provided, it must match the value + currently stored in EC2. If not, the tag will not + be removed. + """ + if value: + tags = {key : value} + else: + tags = [key] + status = self.connection.delete_tags([self.id], tags) + if key in self.tags: + del self.tags[key] diff --git a/backup/src/boto/ec2/elb/__init__.py b/backup/src/boto/ec2/elb/__init__.py new file mode 100644 index 0000000..f4061d3 --- /dev/null +++ b/backup/src/boto/ec2/elb/__init__.py @@ -0,0 +1,427 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This module provides an interface to the Elastic Compute Cloud (EC2) +load balancing service from AWS. +""" +from boto.connection import AWSQueryConnection +from boto.ec2.instanceinfo import InstanceInfo +from boto.ec2.elb.loadbalancer import LoadBalancer +from boto.ec2.elb.instancestate import InstanceState +from boto.ec2.elb.healthcheck import HealthCheck +from boto.regioninfo import RegionInfo +import boto + +RegionData = { + 'us-east-1' : 'elasticloadbalancing.us-east-1.amazonaws.com', + 'us-west-1' : 'elasticloadbalancing.us-west-1.amazonaws.com', + 'eu-west-1' : 'elasticloadbalancing.eu-west-1.amazonaws.com', + 'ap-southeast-1' : 'elasticloadbalancing.ap-southeast-1.amazonaws.com'} + +def regions(): + """ + Get all available regions for the SDB service. + + :rtype: list + :return: A list of :class:`boto.RegionInfo` instances + """ + regions = [] + for region_name in RegionData: + region = RegionInfo(name=region_name, + endpoint=RegionData[region_name], + connection_cls=ELBConnection) + regions.append(region) + return regions + +def connect_to_region(region_name, **kw_params): + """ + Given a valid region name, return a + :class:`boto.ec2.elb.ELBConnection`. + + :param str region_name: The name of the region to connect to. + + :rtype: :class:`boto.ec2.ELBConnection` or ``None`` + :return: A connection to the given region, or None if an invalid region + name is given + """ + for region in regions(): + if region.name == region_name: + return region.connect(**kw_params) + return None + +class ELBConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'elb_version', '2010-07-01') + DefaultRegionName = boto.config.get('Boto', 'elb_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'elb_region_endpoint', + 'elasticloadbalancing.amazonaws.com') + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + """ + Init method to create a new connection to EC2 Load Balancing Service. + + B{Note:} The region argument is overridden by the region specified in + the boto configuration file. + """ + if not region: + region = RegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['ec2'] + + def build_list_params(self, params, items, label): + if isinstance(items, str): + items = [items] + for i in range(1, len(items)+1): + params[label % i] = items[i-1] + + def get_all_load_balancers(self, load_balancer_names=None): + """ + Retrieve all load balancers associated with your account. + + :type load_balancer_names: list + :param load_balancer_names: An optional list of load balancer names + + :rtype: list + :return: A list of :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {} + if load_balancer_names: + self.build_list_params(params, load_balancer_names, 'LoadBalancerNames.member.%d') + return self.get_list('DescribeLoadBalancers', params, [('member', LoadBalancer)]) + + + def create_load_balancer(self, name, zones, listeners): + """ + Create a new load balancer for your account. + + :type name: string + :param name: The mnemonic name associated with the new load balancer + + :type zones: List of strings + :param zones: The names of the availability zone(s) to add. + + :type listeners: List of tuples + :param listeners: Each tuple contains three or four values, + (LoadBalancerPortNumber, InstancePortNumber, Protocol, + [SSLCertificateId]) + where LoadBalancerPortNumber and InstancePortNumber are + integer values between 1 and 65535, Protocol is a + string containing either 'TCP', 'HTTP' or 'HTTPS'; + SSLCertificateID is the ARN of a AWS AIM certificate, + and must be specified when doing HTTPS. + + :rtype: :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + :return: The newly created :class:`boto.ec2.elb.loadbalancer.LoadBalancer` + """ + params = {'LoadBalancerName' : name} + for i in range(0, len(listeners)): + params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] + params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] + params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] + if listeners[i][2]=='HTTPS': + params['Listeners.member.%d.SSLCertificateId' % (i+1)] = listeners[i][3] + self.build_list_params(params, zones, 'AvailabilityZones.member.%d') + load_balancer = self.get_object('CreateLoadBalancer', params, LoadBalancer) + load_balancer.name = name + load_balancer.listeners = listeners + load_balancer.availability_zones = zones + return load_balancer + + def create_load_balancer_listeners(self, name, listeners): + """ + Creates a Listener (or group of listeners) for an existing Load Balancer + + :type name: string + :param name: The name of the load balancer to create the listeners for + + :type listeners: List of tuples + :param listeners: Each tuple contains three values, + (LoadBalancerPortNumber, InstancePortNumber, Protocol, + [SSLCertificateId]) + where LoadBalancerPortNumber and InstancePortNumber are + integer values between 1 and 65535, Protocol is a + string containing either 'TCP', 'HTTP' or 'HTTPS'; + SSLCertificateID is the ARN of a AWS AIM certificate, + and must be specified when doing HTTPS. + + :return: The status of the request + """ + params = {'LoadBalancerName' : name} + for i in range(0, len(listeners)): + params['Listeners.member.%d.LoadBalancerPort' % (i+1)] = listeners[i][0] + params['Listeners.member.%d.InstancePort' % (i+1)] = listeners[i][1] + params['Listeners.member.%d.Protocol' % (i+1)] = listeners[i][2] + if listeners[i][2]=='HTTPS': + params['Listeners.member.%d.SSLCertificateId' % (i+1)] = listeners[i][3] + return self.get_status('CreateLoadBalancerListeners', params) + + + def delete_load_balancer(self, name): + """ + Delete a Load Balancer from your account. + + :type name: string + :param name: The name of the Load Balancer to delete + """ + params = {'LoadBalancerName': name} + return self.get_status('DeleteLoadBalancer', params) + + def delete_load_balancer_listeners(self, name, ports): + """ + Deletes a load balancer listener (or group of listeners) + + :type name: string + :param name: The name of the load balancer to create the listeners for + + :type ports: List int + :param ports: Each int represents the port on the ELB to be removed + + :return: The status of the request + """ + params = {'LoadBalancerName' : name} + for i in range(0, len(ports)): + params['LoadBalancerPorts.member.%d' % (i+1)] = ports[i] + return self.get_status('DeleteLoadBalancerListeners', params) + + + + def enable_availability_zones(self, load_balancer_name, zones_to_add): + """ + Add availability zones to an existing Load Balancer + All zones must be in the same region as the Load Balancer + Adding zones that are already registered with the Load Balancer + has no effect. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to add. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_add, 'AvailabilityZones.member.%d') + return self.get_list('EnableAvailabilityZonesForLoadBalancer', params, None) + + def disable_availability_zones(self, load_balancer_name, zones_to_remove): + """ + Remove availability zones from an existing Load Balancer. + All zones must be in the same region as the Load Balancer. + Removing zones that are not registered with the Load Balancer + has no effect. + You cannot remove all zones from an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type zones: List of strings + :param zones: The name of the zone(s) to remove. + + :rtype: List of strings + :return: An updated list of zones for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, zones_to_remove, 'AvailabilityZones.member.%d') + return self.get_list('DisableAvailabilityZonesForLoadBalancer', params, None) + + def register_instances(self, load_balancer_name, instances): + """ + Add new Instances to an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to add. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('RegisterInstancesWithLoadBalancer', params, [('member', InstanceInfo)]) + + def deregister_instances(self, load_balancer_name, instances): + """ + Remove Instances from an existing Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances to remove. + + :rtype: List of strings + :return: An updated list of instances for this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('DeregisterInstancesFromLoadBalancer', params, [('member', InstanceInfo)]) + + def describe_instance_health(self, load_balancer_name, instances=None): + """ + Get current state of all Instances registered to an Load Balancer. + + :type load_balancer_name: string + :param load_balancer_name: The name of the Load Balancer + + :type instances: List of strings + :param instances: The instance ID's of the EC2 instances + to return status for. If not provided, + the state of all instances will be returned. + + :rtype: List of :class:`boto.ec2.elb.instancestate.InstanceState` + :return: list of state info for instances in this Load Balancer. + + """ + params = {'LoadBalancerName' : load_balancer_name} + if instances: + self.build_list_params(params, instances, 'Instances.member.%d.InstanceId') + return self.get_list('DescribeInstanceHealth', params, [('member', InstanceState)]) + + def configure_health_check(self, name, health_check): + """ + Define a health check for the EndPoints. + + :type name: string + :param name: The mnemonic name associated with the new access point + + :type health_check: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :param health_check: A HealthCheck object populated with the desired + values. + + :rtype: :class:`boto.ec2.elb.healthcheck.HealthCheck` + :return: The updated :class:`boto.ec2.elb.healthcheck.HealthCheck` + """ + params = {'LoadBalancerName' : name, + 'HealthCheck.Timeout' : health_check.timeout, + 'HealthCheck.Target' : health_check.target, + 'HealthCheck.Interval' : health_check.interval, + 'HealthCheck.UnhealthyThreshold' : health_check.unhealthy_threshold, + 'HealthCheck.HealthyThreshold' : health_check.healthy_threshold} + return self.get_object('ConfigureHealthCheck', params, HealthCheck) + + def set_lb_listener_SSL_certificate(self, lb_name, lb_port, ssl_certificate_id): + """ + Sets the certificate that terminates the specified listener's SSL + connections. The specified certificate replaces any prior certificate + that was used on the same LoadBalancer and port. + """ + params = { + 'LoadBalancerName' : lb_name, + 'LoadBalancerPort' : lb_port, + 'SSLCertificateId' : ssl_certificate_id, + } + return self.get_status('SetLoadBalancerListenerSSLCertificate', params) + + def create_app_cookie_stickiness_policy(self, name, lb_name, policy_name): + """ + Generates a stickiness policy with sticky session lifetimes that follow + that of an application-generated cookie. This policy can only be + associated with HTTP listeners. + + This policy is similar to the policy created by + CreateLBCookieStickinessPolicy, except that the lifetime of the special + Elastic Load Balancing cookie follows the lifetime of the + application-generated cookie specified in the policy configuration. The + load balancer only inserts a new stickiness cookie when the application + response includes a new application cookie. + + If the application cookie is explicitly removed or expires, the session + stops being sticky until a new application cookie is issued. + """ + params = { + 'CookieName' : name, + 'LoadBalancerName' : lb_name, + 'PolicyName' : policy_name, + } + return self.get_status('CreateAppCookieStickinessPolicy', params) + + def create_lb_cookie_stickiness_policy(self, cookie_expiration_period, lb_name, policy_name): + """ + Generates a stickiness policy with sticky session lifetimes controlled + by the lifetime of the browser (user-agent) or a specified expiration + period. This policy can only be associated only with HTTP listeners. + + When a load balancer implements this policy, the load balancer uses a + special cookie to track the backend server instance for each request. + When the load balancer receives a request, it first checks to see if + this cookie is present in the request. If so, the load balancer sends + the request to the application server specified in the cookie. If not, + the load balancer sends the request to a server that is chosen based on + the existing load balancing algorithm. + + A cookie is inserted into the response for binding subsequent requests + from the same user to that server. The validity of the cookie is based + on the cookie expiration time, which is specified in the policy + configuration. + """ + params = { + 'CookieExpirationPeriod' : cookie_expiration_period, + 'LoadBalancerName' : lb_name, + 'PolicyName' : policy_name, + } + return self.get_status('CreateLBCookieStickinessPolicy', params) + + def delete_lb_policy(self, lb_name, policy_name): + """ + Deletes a policy from the LoadBalancer. The specified policy must not + be enabled for any listeners. + """ + params = { + 'LoadBalancerName' : lb_name, + 'PolicyName' : policy_name, + } + return self.get_status('DeleteLoadBalancerPolicy', params) + + def set_lb_policies_of_listener(self, lb_name, lb_port, policies): + """ + Associates, updates, or disables a policy with a listener on the load + balancer. Currently only zero (0) or one (1) policy can be associated + with a listener. + """ + params = { + 'LoadBalancerName' : lb_name, + 'LoadBalancerPort' : lb_port, + } + self.build_list_params(params, policies, 'PolicyNames.member.%d') + return self.get_status('SetLoadBalancerPoliciesOfListener', params) + + diff --git a/backup/src/boto/ec2/elb/healthcheck.py b/backup/src/boto/ec2/elb/healthcheck.py new file mode 100644 index 0000000..5a3edbc --- /dev/null +++ b/backup/src/boto/ec2/elb/healthcheck.py @@ -0,0 +1,68 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class HealthCheck(object): + """ + Represents an EC2 Access Point Health Check + """ + + def __init__(self, access_point=None, interval=30, target=None, + healthy_threshold=3, timeout=5, unhealthy_threshold=5): + self.access_point = access_point + self.interval = interval + self.target = target + self.healthy_threshold = healthy_threshold + self.timeout = timeout + self.unhealthy_threshold = unhealthy_threshold + + def __repr__(self): + return 'HealthCheck:%s' % self.target + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Interval': + self.interval = int(value) + elif name == 'Target': + self.target = value + elif name == 'HealthyThreshold': + self.healthy_threshold = int(value) + elif name == 'Timeout': + self.timeout = int(value) + elif name == 'UnhealthyThreshold': + self.unhealthy_threshold = int(value) + else: + setattr(self, name, value) + + def update(self): + if not self.access_point: + return + + new_hc = self.connection.configure_health_check(self.access_point, + self) + self.interval = new_hc.interval + self.target = new_hc.target + self.healthy_threshold = new_hc.healthy_threshold + self.unhealthy_threshold = new_hc.unhealthy_threshold + self.timeout = new_hc.timeout + + diff --git a/backup/src/boto/ec2/elb/instancestate.py b/backup/src/boto/ec2/elb/instancestate.py new file mode 100644 index 0000000..4a9b0d4 --- /dev/null +++ b/backup/src/boto/ec2/elb/instancestate.py @@ -0,0 +1,54 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceState(object): + """ + Represents the state of an EC2 Load Balancer Instance + """ + + def __init__(self, load_balancer=None, description=None, + state=None, instance_id=None, reason_code=None): + self.load_balancer = load_balancer + self.description = description + self.state = state + self.instance_id = instance_id + self.reason_code = reason_code + + def __repr__(self): + return 'InstanceState:(%s,%s)' % (self.instance_id, self.state) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Description': + self.description = value + elif name == 'State': + self.state = value + elif name == 'InstanceId': + self.instance_id = value + elif name == 'ReasonCode': + self.reason_code = value + else: + setattr(self, name, value) + + + diff --git a/backup/src/boto/ec2/elb/listelement.py b/backup/src/boto/ec2/elb/listelement.py new file mode 100644 index 0000000..5be4599 --- /dev/null +++ b/backup/src/boto/ec2/elb/listelement.py @@ -0,0 +1,31 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ListElement(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'member': + self.append(value) + + diff --git a/backup/src/boto/ec2/elb/listener.py b/backup/src/boto/ec2/elb/listener.py new file mode 100644 index 0000000..a8807c0 --- /dev/null +++ b/backup/src/boto/ec2/elb/listener.py @@ -0,0 +1,71 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Listener(object): + """ + Represents an EC2 Load Balancer Listener tuple + """ + + def __init__(self, load_balancer=None, load_balancer_port=0, + instance_port=0, protocol='', ssl_certificate_id=None): + self.load_balancer = load_balancer + self.load_balancer_port = load_balancer_port + self.instance_port = instance_port + self.protocol = protocol + self.ssl_certificate_id = ssl_certificate_id + + def __repr__(self): + r = "(%d, %d, '%s'" % (self.load_balancer_port, self.instance_port, self.protocol) + if self.ssl_certificate_id: + r += ', %s' % (self.ssl_certificate_id) + r += ')' + return r + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerPort': + self.load_balancer_port = int(value) + elif name == 'InstancePort': + self.instance_port = int(value) + elif name == 'Protocol': + self.protocol = value + elif name == 'SSLCertificateId': + self.ssl_certificate_id = value + else: + setattr(self, name, value) + + def get_tuple(self): + return self.load_balancer_port, self.instance_port, self.protocol + + def __getitem__(self, key): + if key == 0: + return self.load_balancer_port + if key == 1: + return self.instance_port + if key == 2: + return self.protocol + raise KeyError + + + + diff --git a/backup/src/boto/ec2/elb/loadbalancer.py b/backup/src/boto/ec2/elb/loadbalancer.py new file mode 100644 index 0000000..9759952 --- /dev/null +++ b/backup/src/boto/ec2/elb/loadbalancer.py @@ -0,0 +1,182 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.elb.healthcheck import HealthCheck +from boto.ec2.elb.listener import Listener +from boto.ec2.elb.listelement import ListElement +from boto.ec2.elb.policies import Policies +from boto.ec2.instanceinfo import InstanceInfo +from boto.resultset import ResultSet + +class LoadBalancer(object): + """ + Represents an EC2 Load Balancer + """ + + def __init__(self, connection=None, name=None, endpoints=None): + self.connection = connection + self.name = name + self.listeners = None + self.health_check = None + self.policies = None + self.dns_name = None + self.created_time = None + self.instances = None + self.availability_zones = ListElement() + + def __repr__(self): + return 'LoadBalancer:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'HealthCheck': + self.health_check = HealthCheck(self) + return self.health_check + elif name == 'ListenerDescriptions': + self.listeners = ResultSet([('member', Listener)]) + return self.listeners + elif name == 'AvailabilityZones': + return self.availability_zones + elif name == 'Instances': + self.instances = ResultSet([('member', InstanceInfo)]) + return self.instances + elif name == 'Policies': + self.policies = Policies(self) + return self.policies + else: + return None + + def endElement(self, name, value, connection): + if name == 'LoadBalancerName': + self.name = value + elif name == 'DNSName': + self.dns_name = value + elif name == 'CreatedTime': + self.created_time = value + elif name == 'InstanceId': + self.instances.append(value) + else: + setattr(self, name, value) + + def enable_zones(self, zones): + """ + Enable availability zones to this Access Point. + All zones must be in the same region as the Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.enable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def disable_zones(self, zones): + """ + Disable availability zones from this Access Point. + + :type zones: string or List of strings + :param zones: The name of the zone(s) to add. + + """ + if isinstance(zones, str) or isinstance(zones, unicode): + zones = [zones] + new_zones = self.connection.disable_availability_zones(self.name, zones) + self.availability_zones = new_zones + + def register_instances(self, instances): + """ + Add instances to this Load Balancer + All instances must be in the same region as the Load Balancer. + Adding endpoints that are already registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.register_instances(self.name, instances) + self.instances = new_instances + + def deregister_instances(self, instances): + """ + Remove instances from this Load Balancer. + Removing instances that are not registered with the Load Balancer + has no effect. + + :type zones: string or List of instance id's + :param zones: The name of the endpoint(s) to add. + + """ + if isinstance(instances, str) or isinstance(instances, unicode): + instances = [instances] + new_instances = self.connection.deregister_instances(self.name, instances) + self.instances = new_instances + + def delete(self): + """ + Delete this load balancer + """ + return self.connection.delete_load_balancer(self.name) + + def configure_health_check(self, health_check): + return self.connection.configure_health_check(self.name, health_check) + + def get_instance_health(self, instances=None): + return self.connection.describe_instance_health(self.name, instances) + + def create_listeners(self, listeners): + return self.connection.create_load_balancer_listeners(self.name, listeners) + + def create_listener(self, inPort, outPort=None, proto="tcp"): + if outPort == None: + outPort = inPort + return self.create_listeners([(inPort, outPort, proto)]) + + def delete_listeners(self, listeners): + return self.connection.delete_load_balancer_listeners(self.name, listeners) + + def delete_listener(self, inPort, outPort=None, proto="tcp"): + if outPort == None: + outPort = inPort + return self.delete_listeners([(inPort, outPort, proto)]) + + def delete_policy(self, policy_name): + """ + Deletes a policy from the LoadBalancer. The specified policy must not + be enabled for any listeners. + """ + return self.connection.delete_lb_policy(self.name, policy_name) + + def set_policies_of_listener(self, lb_port, policies): + return self.connection.set_lb_policies_of_listener(self.name, lb_port, policies) + + def create_cookie_stickiness_policy(self, cookie_expiration_period, policy_name): + return self.connection.create_lb_cookie_stickiness_policy(cookie_expiration_period, self.name, policy_name) + + def create_app_cookie_stickiness_policy(self, name, policy_name): + return self.connection.create_app_cookie_stickiness_policy(name, self.name, policy_name) + + def set_listener_SSL_certificate(self, lb_port, ssl_certificate_id): + return self.connection.set_lb_listener_SSL_certificate(self.name, lb_port, ssl_certificate_id) + diff --git a/backup/src/boto/ec2/elb/policies.py b/backup/src/boto/ec2/elb/policies.py new file mode 100644 index 0000000..428ce72 --- /dev/null +++ b/backup/src/boto/ec2/elb/policies.py @@ -0,0 +1,82 @@ +# Copyright (c) 2010 Reza Lotun http://reza.lotun.name +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.resultset import ResultSet + + +class AppCookieStickinessPolicy(object): + def __init__(self, connection=None): + self.cookie_name = None + self.policy_name = None + + def __repr__(self): + return 'AppCookieStickiness(%s, %s)' % (self.policy_name, self.cookie_name) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'CookieName': + self.cookie_name = value + elif name == 'PolicyName': + self.policy_name = value + + +class LBCookieStickinessPolicy(object): + def __init__(self, connection=None): + self.policy_name = None + self.cookie_expiration_period = None + + def __repr__(self): + return 'LBCookieStickiness(%s, %s)' % (self.policy_name, self.cookie_expiration_period) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'CookieExpirationPeriod': + self.cookie_expiration_period = value + elif name == 'PolicyName': + self.policy_name = value + + +class Policies(object): + """ + ELB Policies + """ + def __init__(self, connection=None): + self.connection = connection + self.app_cookie_stickiness_policies = None + self.lb_cookie_stickiness_policies = None + + def __repr__(self): + return 'Policies(AppCookieStickiness%s, LBCookieStickiness%s)' % (self.app_cookie_stickiness_policies, + self.lb_cookie_stickiness_policies) + + def startElement(self, name, attrs, connection): + if name == 'AppCookieStickinessPolicies': + self.app_cookie_stickiness_policies = ResultSet([('member', AppCookieStickinessPolicy)]) + elif name == 'LBCookieStickinessPolicies': + self.lb_cookie_stickiness_policies = ResultSet([('member', LBCookieStickinessPolicy)]) + + def endElement(self, name, value, connection): + return + diff --git a/backup/src/boto/ec2/image.py b/backup/src/boto/ec2/image.py new file mode 100644 index 0000000..a85fba0 --- /dev/null +++ b/backup/src/boto/ec2/image.py @@ -0,0 +1,322 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object, TaggedEC2Object +from boto.ec2.blockdevicemapping import BlockDeviceMapping + +class ProductCodes(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'productCode': + self.append(value) + +class Image(TaggedEC2Object): + """ + Represents an EC2 Image + """ + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.location = None + self.state = None + self.ownerId = None # for backwards compatibility + self.owner_id = None + self.owner_alias = None + self.is_public = False + self.architecture = None + self.platform = None + self.type = None + self.kernel_id = None + self.ramdisk_id = None + self.name = None + self.description = None + self.product_codes = ProductCodes() + self.block_device_mapping = None + self.root_device_type = None + self.root_device_name = None + self.virtualization_type = None + self.hypervisor = None + self.instance_lifecycle = None + + def __repr__(self): + return 'Image:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.id = value + elif name == 'imageLocation': + self.location = value + elif name == 'imageState': + self.state = value + elif name == 'imageOwnerId': + self.ownerId = value # for backwards compatibility + self.owner_id = value + elif name == 'isPublic': + if value == 'false': + self.is_public = False + elif value == 'true': + self.is_public = True + else: + raise Exception( + 'Unexpected value of isPublic %s for image %s'%( + value, + self.id + ) + ) + elif name == 'architecture': + self.architecture = value + elif name == 'imageType': + self.type = value + elif name == 'kernelId': + self.kernel_id = value + elif name == 'ramdiskId': + self.ramdisk_id = value + elif name == 'imageOwnerAlias': + self.owner_alias = value + elif name == 'platform': + self.platform = value + elif name == 'name': + self.name = value + elif name == 'description': + self.description = value + elif name == 'rootDeviceType': + self.root_device_type = value + elif name == 'rootDeviceName': + self.root_device_name = value + elif name == 'virtualizationType': + self.virtualization_type = value + elif name == 'hypervisor': + self.hypervisor = value + elif name == 'instanceLifecycle': + self.instance_lifecycle = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.__dict__.update(updated.__dict__) + + def update(self, validate=False): + """ + Update the image's state information by making a call to fetch + the current image attributes from the service. + + :type validate: bool + :param validate: By default, if EC2 returns no data about the + image the update method returns quietly. If + the validate param is True, however, it will + raise a ValueError exception if no data is + returned from EC2. + """ + rs = self.connection.get_all_images([self.id]) + if len(rs) > 0: + img = rs[0] + if img.id == self.id: + self._update(img) + elif validate: + raise ValueError('%s is not a valid Image ID' % self.id) + return self.state + + def run(self, min_count=1, max_count=1, key_name=None, + security_groups=None, user_data=None, + addressing_type=None, instance_type='m1.small', placement=None, + kernel_id=None, ramdisk_id=None, + monitoring_enabled=False, subnet_id=None, + block_device_map=None, + disable_api_termination=False, + instance_initiated_shutdown_behavior=None, + private_ip_address=None, + placement_group=None): + """ + Runs this instance. + + :type min_count: int + :param min_count: The minimum number of instances to start + + :type max_count: int + :param max_count: The maximum number of instances to start + + :type key_name: string + :param key_name: The name of the keypair to run this instance with. + + :type security_groups: + :param security_groups: + + :type user_data: + :param user_data: + + :type addressing_type: + :param daddressing_type: + + :type instance_type: string + :param instance_type: The type of instance to run. Current choices are: + m1.small | m1.large | m1.xlarge | c1.medium | + c1.xlarge | m2.xlarge | m2.2xlarge | + m2.4xlarge | cc1.4xlarge + + :type placement: string + :param placement: The availability zone in which to launch the instances + + :type kernel_id: string + :param kernel_id: The ID of the kernel with which to launch the instances + + :type ramdisk_id: string + :param ramdisk_id: The ID of the RAM disk with which to launch the instances + + :type monitoring_enabled: bool + :param monitoring_enabled: Enable CloudWatch monitoring on the instance. + + :type subnet_id: string + :param subnet_id: The subnet ID within which to launch the instances for VPC. + + :type private_ip_address: string + :param private_ip_address: If you're using VPC, you can optionally use + this parameter to assign the instance a + specific available IP address from the + subnet (e.g., 10.0.0.25). + + :type block_device_map: :class:`boto.ec2.blockdevicemapping.BlockDeviceMapping` + :param block_device_map: A BlockDeviceMapping data structure + describing the EBS volumes associated + with the Image. + + :type disable_api_termination: bool + :param disable_api_termination: If True, the instances will be locked + and will not be able to be terminated + via the API. + + :type instance_initiated_shutdown_behavior: string + :param instance_initiated_shutdown_behavior: Specifies whether the instance's + EBS volumes are stopped (i.e. detached) + or terminated (i.e. deleted) when + the instance is shutdown by the + owner. Valid values are: + stop | terminate + + :type placement_group: string + :param placement_group: If specified, this is the name of the placement + group in which the instance(s) will be launched. + + :rtype: Reservation + :return: The :class:`boto.ec2.instance.Reservation` associated with the request for machines + """ + return self.connection.run_instances(self.id, min_count, max_count, + key_name, security_groups, + user_data, addressing_type, + instance_type, placement, + kernel_id, ramdisk_id, + monitoring_enabled, subnet_id, + block_device_map, disable_api_termination, + instance_initiated_shutdown_behavior, + private_ip_address, + placement_group) + + def deregister(self): + return self.connection.deregister_image(self.id) + + def get_launch_permissions(self): + img_attrs = self.connection.get_image_attribute(self.id, + 'launchPermission') + return img_attrs.attrs + + def set_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'add', + user_ids, + group_names) + + def remove_launch_permissions(self, user_ids=None, group_names=None): + return self.connection.modify_image_attribute(self.id, + 'launchPermission', + 'remove', + user_ids, + group_names) + + def reset_launch_attributes(self): + return self.connection.reset_image_attribute(self.id, + 'launchPermission') + + def get_kernel(self): + img_attrs =self.connection.get_image_attribute(self.id, 'kernel') + return img_attrs.kernel + + def get_ramdisk(self): + img_attrs = self.connection.get_image_attribute(self.id, 'ramdisk') + return img_attrs.ramdisk + +class ImageAttribute: + + def __init__(self, parent=None): + self.name = None + self.kernel = None + self.ramdisk = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + if name == 'blockDeviceMapping': + self.attrs['block_device_mapping'] = BlockDeviceMapping() + return self.attrs['block_device_mapping'] + else: + return None + + def endElement(self, name, value, connection): + if name == 'launchPermission': + self.name = 'launch_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'productCode': + if self.attrs.has_key('product_codes'): + self.attrs['product_codes'].append(value) + else: + self.attrs['product_codes'] = [value] + elif name == 'imageId': + self.image_id = value + elif name == 'kernel': + self.kernel = value + elif name == 'ramdisk': + self.ramdisk = value + else: + setattr(self, name, value) diff --git a/backup/src/boto/ec2/instance.py b/backup/src/boto/ec2/instance.py new file mode 100644 index 0000000..9e8aacf --- /dev/null +++ b/backup/src/boto/ec2/instance.py @@ -0,0 +1,394 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Instance +""" +import boto +from boto.ec2.ec2object import EC2Object, TaggedEC2Object +from boto.resultset import ResultSet +from boto.ec2.address import Address +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.image import ProductCodes +import base64 + +class Reservation(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.owner_id = None + self.groups = [] + self.instances = [] + + def __repr__(self): + return 'Reservation:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'instancesSet': + self.instances = ResultSet([('item', Instance)]) + return self.instances + elif name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + else: + return None + + def endElement(self, name, value, connection): + if name == 'reservationId': + self.id = value + elif name == 'ownerId': + self.owner_id = value + else: + setattr(self, name, value) + + def stop_all(self): + for instance in self.instances: + instance.stop() + +class Instance(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.dns_name = None + self.public_dns_name = None + self.private_dns_name = None + self.state = None + self.state_code = None + self.key_name = None + self.shutdown_state = None + self.previous_state = None + self.instance_type = None + self.instance_class = None + self.launch_time = None + self.image_id = None + self.placement = None + self.kernel = None + self.ramdisk = None + self.product_codes = ProductCodes() + self.ami_launch_index = None + self.monitored = False + self.instance_class = None + self.spot_instance_request_id = None + self.subnet_id = None + self.vpc_id = None + self.private_ip_address = None + self.ip_address = None + self.requester_id = None + self._in_monitoring_element = False + self.persistent = False + self.root_device_name = None + self.root_device_type = None + self.block_device_mapping = None + self.state_reason = None + self.group_name = None + self.client_token = None + + def __repr__(self): + return 'Instance:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + elif name == 'productCodes': + return self.product_codes + elif name == 'stateReason': + self.state_reason = StateReason() + return self.state_reason + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.id = value + elif name == 'imageId': + self.image_id = value + elif name == 'dnsName' or name == 'publicDnsName': + self.dns_name = value # backwards compatibility + self.public_dns_name = value + elif name == 'privateDnsName': + self.private_dns_name = value + elif name == 'keyName': + self.key_name = value + elif name == 'amiLaunchIndex': + self.ami_launch_index = value + elif name == 'shutdownState': + self.shutdown_state = value + elif name == 'previousState': + self.previous_state = value + elif name == 'name': + self.state = value + elif name == 'code': + try: + self.state_code = int(value) + except ValueError: + boto.log.warning('Error converting code (%s) to int' % value) + self.state_code = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'instanceClass': + self.instance_class = value + elif name == 'rootDeviceName': + self.root_device_name = value + elif name == 'rootDeviceType': + self.root_device_type = value + elif name == 'launchTime': + self.launch_time = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + elif name == 'instanceClass': + self.instance_class = value + elif name == 'spotInstanceRequestId': + self.spot_instance_request_id = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'vpcId': + self.vpc_id = value + elif name == 'privateIpAddress': + self.private_ip_address = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'requesterId': + self.requester_id = value + elif name == 'persistent': + if value == 'true': + self.persistent = True + else: + self.persistent = False + elif name == 'groupName': + if self._in_monitoring_element: + self.group_name = value + elif name == 'clientToken': + self.client_token = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.__dict__.update(updated.__dict__) + + def update(self, validate=False): + """ + Update the instance's state information by making a call to fetch + the current instance attributes from the service. + + :type validate: bool + :param validate: By default, if EC2 returns no data about the + instance the update method returns quietly. If + the validate param is True, however, it will + raise a ValueError exception if no data is + returned from EC2. + """ + rs = self.connection.get_all_instances([self.id]) + if len(rs) > 0: + r = rs[0] + for i in r.instances: + if i.id == self.id: + self._update(i) + elif validate: + raise ValueError('%s is not a valid Instance ID' % self.id) + return self.state + + def terminate(self): + """ + Terminate the instance + """ + rs = self.connection.terminate_instances([self.id]) + self._update(rs[0]) + + def stop(self, force=False): + """ + Stop the instance + + :type force: bool + :param force: Forces the instance to stop + + :rtype: list + :return: A list of the instances stopped + """ + rs = self.connection.stop_instances([self.id]) + self._update(rs[0]) + + def start(self): + """ + Start the instance. + """ + rs = self.connection.start_instances([self.id]) + self._update(rs[0]) + + def reboot(self): + return self.connection.reboot_instances([self.id]) + + def get_console_output(self): + """ + Retrieves the console output for the instance. + + :rtype: :class:`boto.ec2.instance.ConsoleOutput` + :return: The console output as a ConsoleOutput object + """ + return self.connection.get_console_output(self.id) + + def confirm_product(self, product_code): + return self.connection.confirm_product_instance(self.id, product_code) + + def use_ip(self, ip_address): + if isinstance(ip_address, Address): + ip_address = ip_address.public_ip + return self.connection.associate_address(self.id, ip_address) + + def monitor(self): + return self.connection.monitor_instance(self.id) + + def unmonitor(self): + return self.connection.unmonitor_instance(self.id) + + def get_attribute(self, attribute): + """ + Gets an attribute from this instance. + + :type attribute: string + :param attribute: The attribute you need information about + Valid choices are: + instanceType|kernel|ramdisk|userData| + disableApiTermination| + instanceInitiatedShutdownBehavior| + rootDeviceName|blockDeviceMapping + + :rtype: :class:`boto.ec2.image.InstanceAttribute` + :return: An InstanceAttribute object representing the value of the + attribute requested + """ + return self.connection.get_instance_attribute(self.id, attribute) + + def modify_attribute(self, attribute, value): + """ + Changes an attribute of this instance + + :type attribute: string + :param attribute: The attribute you wish to change. + AttributeName - Expected value (default) + instanceType - A valid instance type (m1.small) + kernel - Kernel ID (None) + ramdisk - Ramdisk ID (None) + userData - Base64 encoded String (None) + disableApiTermination - Boolean (true) + instanceInitiatedShutdownBehavior - stop|terminate + rootDeviceName - device name (None) + + :type value: string + :param value: The new value for the attribute + + :rtype: bool + :return: Whether the operation succeeded or not + """ + return self.connection.modify_instance_attribute(self.id, attribute, + value) + + def reset_attribute(self, attribute): + """ + Resets an attribute of this instance to its default value. + + :type attribute: string + :param attribute: The attribute to reset. Valid values are: + kernel|ramdisk + + :rtype: bool + :return: Whether the operation succeeded or not + """ + return self.connection.reset_instance_attribute(self.id, attribute) + +class Group: + + def __init__(self, parent=None): + self.id = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'groupId': + self.id = value + else: + setattr(self, name, value) + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class InstanceAttribute(dict): + + def __init__(self, parent=None): + dict.__init__(self) + self._current_value = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'value': + self._current_value = value + else: + self[name] = self._current_value + +class StateReason(dict): + + def __init__(self, parent=None): + dict.__init__(self) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name != 'stateReason': + self[name] = value + diff --git a/backup/src/boto/ec2/instanceinfo.py b/backup/src/boto/ec2/instanceinfo.py new file mode 100644 index 0000000..6efbaed --- /dev/null +++ b/backup/src/boto/ec2/instanceinfo.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class InstanceInfo(object): + """ + Represents an EC2 Instance status response from CloudWatch + """ + + def __init__(self, connection=None, id=None, state=None): + self.connection = connection + self.id = id + self.state = state + + def __repr__(self): + return 'InstanceInfo:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId' or name == 'InstanceId': + self.id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + + diff --git a/backup/src/boto/ec2/keypair.py b/backup/src/boto/ec2/keypair.py new file mode 100644 index 0000000..d08e5ce --- /dev/null +++ b/backup/src/boto/ec2/keypair.py @@ -0,0 +1,111 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Keypair +""" + +import os +from boto.ec2.ec2object import EC2Object +from boto.exception import BotoClientError + +class KeyPair(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.fingerprint = None + self.material = None + + def __repr__(self): + return 'KeyPair:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'keyName': + self.name = value + elif name == 'keyFingerprint': + self.fingerprint = value + elif name == 'keyMaterial': + self.material = value + else: + setattr(self, name, value) + + def delete(self): + """ + Delete the KeyPair. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.delete_key_pair(self.name) + + def save(self, directory_path): + """ + Save the material (the unencrypted PEM encoded RSA private key) + of a newly created KeyPair to a local file. + + :type directory_path: string + :param directory_path: The fully qualified path to the directory + in which the keypair will be saved. The + keypair file will be named using the name + of the keypair as the base name and .pem + for the file extension. If a file of that + name already exists in the directory, an + exception will be raised and the old file + will not be overwritten. + + :rtype: bool + :return: True if successful. + """ + if self.material: + file_path = os.path.join(directory_path, '%s.pem' % self.name) + if os.path.exists(file_path): + raise BotoClientError('%s already exists, it will not be overwritten' % file_path) + fp = open(file_path, 'wb') + fp.write(self.material) + fp.close() + return True + else: + raise BotoClientError('KeyPair contains no material') + + def copy_to_region(self, region): + """ + Create a new key pair of the same new in another region. + Note that the new key pair will use a different ssh + cert than the this key pair. After doing the copy, + you will need to save the material associated with the + new key pair (use the save method) to a local file. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :rtype: :class:`boto.ec2.keypair.KeyPair` + :return: The new key pair + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + kp = rconn.create_key_pair(self.name) + return kp + + + diff --git a/backup/src/boto/ec2/launchspecification.py b/backup/src/boto/ec2/launchspecification.py new file mode 100644 index 0000000..a574a38 --- /dev/null +++ b/backup/src/boto/ec2/launchspecification.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a launch specification for Spot instances. +""" + +from boto.ec2.ec2object import EC2Object +from boto.resultset import ResultSet +from boto.ec2.blockdevicemapping import BlockDeviceMapping +from boto.ec2.instance import Group + +class GroupList(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'groupId': + self.append(value) + +class LaunchSpecification(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.key_name = None + self.instance_type = None + self.image_id = None + self.groups = [] + self.placement = None + self.kernel = None + self.ramdisk = None + self.monitored = False + self.subnet_id = None + self._in_monitoring_element = False + self.block_device_mapping = None + + def __repr__(self): + return 'LaunchSpecification(%s)' % self.image_id + + def startElement(self, name, attrs, connection): + if name == 'groupSet': + self.groups = ResultSet([('item', Group)]) + return self.groups + elif name == 'monitoring': + self._in_monitoring_element = True + elif name == 'blockDeviceMapping': + self.block_device_mapping = BlockDeviceMapping() + return self.block_device_mapping + else: + return None + + def endElement(self, name, value, connection): + if name == 'imageId': + self.image_id = value + elif name == 'keyName': + self.key_name = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.placement = value + elif name == 'placement': + pass + elif name == 'kernelId': + self.kernel = value + elif name == 'ramdiskId': + self.ramdisk = value + elif name == 'subnetId': + self.subnet_id = value + elif name == 'state': + if self._in_monitoring_element: + if value == 'enabled': + self.monitored = True + self._in_monitoring_element = False + else: + setattr(self, name, value) + + diff --git a/backup/src/boto/ec2/placementgroup.py b/backup/src/boto/ec2/placementgroup.py new file mode 100644 index 0000000..e1bbea6 --- /dev/null +++ b/backup/src/boto/ec2/placementgroup.py @@ -0,0 +1,51 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +Represents an EC2 Placement Group +""" +from boto.ec2.ec2object import EC2Object +from boto.exception import BotoClientError + +class PlacementGroup(EC2Object): + + def __init__(self, connection=None, name=None, strategy=None, state=None): + EC2Object.__init__(self, connection) + self.name = name + self.strategy = strategy + self.state = state + + def __repr__(self): + return 'PlacementGroup:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'groupName': + self.name = value + elif name == 'strategy': + self.strategy = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_placement_group(self.name) + + diff --git a/backup/src/boto/ec2/regioninfo.py b/backup/src/boto/ec2/regioninfo.py new file mode 100644 index 0000000..0b37b0e --- /dev/null +++ b/backup/src/boto/ec2/regioninfo.py @@ -0,0 +1,34 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.regioninfo import RegionInfo + +class EC2RegionInfo(RegionInfo): + """ + Represents an EC2 Region + """ + + def __init__(self, connection=None, name=None, endpoint=None): + from boto.ec2.connection import EC2Connection + RegionInfo.__init__(self, connection, name, endpoint, + EC2Connection) diff --git a/backup/src/boto/ec2/reservedinstance.py b/backup/src/boto/ec2/reservedinstance.py new file mode 100644 index 0000000..1d35c1d --- /dev/null +++ b/backup/src/boto/ec2/reservedinstance.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.ec2.ec2object import EC2Object + +class ReservedInstancesOffering(EC2Object): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None): + EC2Object.__init__(self, connection) + self.id = id + self.instance_type = instance_type + self.availability_zone = availability_zone + self.duration = duration + self.fixed_price = fixed_price + self.usage_price = usage_price + self.description = description + + def __repr__(self): + return 'ReservedInstanceOffering:%s' % self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'reservedInstancesOfferingId': + self.id = value + elif name == 'instanceType': + self.instance_type = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'duration': + self.duration = value + elif name == 'fixedPrice': + self.fixed_price = value + elif name == 'usagePrice': + self.usage_price = value + elif name == 'productDescription': + self.description = value + else: + setattr(self, name, value) + + def describe(self): + print 'ID=%s' % self.id + print '\tInstance Type=%s' % self.instance_type + print '\tZone=%s' % self.availability_zone + print '\tDuration=%s' % self.duration + print '\tFixed Price=%s' % self.fixed_price + print '\tUsage Price=%s' % self.usage_price + print '\tDescription=%s' % self.description + + def purchase(self, instance_count=1): + return self.connection.purchase_reserved_instance_offering(self.id, instance_count) + +class ReservedInstance(ReservedInstancesOffering): + + def __init__(self, connection=None, id=None, instance_type=None, + availability_zone=None, duration=None, fixed_price=None, + usage_price=None, description=None, + instance_count=None, state=None): + ReservedInstancesOffering.__init__(self, connection, id, instance_type, + availability_zone, duration, fixed_price, + usage_price, description) + self.instance_count = instance_count + self.state = state + + def __repr__(self): + return 'ReservedInstance:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'reservedInstancesId': + self.id = value + if name == 'instanceCount': + self.instance_count = int(value) + elif name == 'state': + self.state = value + else: + ReservedInstancesOffering.endElement(self, name, value, connection) diff --git a/backup/src/boto/ec2/securitygroup.py b/backup/src/boto/ec2/securitygroup.py new file mode 100644 index 0000000..24e08c3 --- /dev/null +++ b/backup/src/boto/ec2/securitygroup.py @@ -0,0 +1,286 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Security Group +""" +from boto.ec2.ec2object import EC2Object +from boto.exception import BotoClientError + +class SecurityGroup(EC2Object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.name = name + self.description = description + self.rules = [] + + def __repr__(self): + return 'SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'item': + self.rules.append(IPPermissions(self)) + return self.rules[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'groupName': + self.name = value + elif name == 'groupDescription': + self.description = value + elif name == 'ipRanges': + pass + elif name == 'return': + if value == 'false': + self.status = False + elif value == 'true': + self.status = True + else: + raise Exception( + 'Unexpected value of status %s for group %s'%( + value, + self.name + ) + ) + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_security_group(self.name) + + def add_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + """ + Add a rule to the SecurityGroup object. Note that this method + only changes the local version of the object. No information + is sent to EC2. + """ + rule = IPPermissions(self) + rule.ip_protocol = ip_protocol + rule.from_port = from_port + rule.to_port = to_port + self.rules.append(rule) + rule.add_grant(src_group_name, src_group_owner_id, cidr_ip) + + def remove_rule(self, ip_protocol, from_port, to_port, + src_group_name, src_group_owner_id, cidr_ip): + """ + Remove a rule to the SecurityGroup object. Note that this method + only changes the local version of the object. No information + is sent to EC2. + """ + target_rule = None + for rule in self.rules: + if rule.ip_protocol == ip_protocol: + if rule.from_port == from_port: + if rule.to_port == to_port: + target_rule = rule + target_grant = None + for grant in rule.grants: + if grant.name == src_group_name: + if grant.owner_id == src_group_owner_id: + if grant.cidr_ip == cidr_ip: + target_grant = grant + if target_grant: + rule.grants.remove(target_grant) + if len(rule.grants) == 0: + self.rules.remove(target_rule) + + def authorize(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + """ + Add a new rule to this security group. + You need to pass in either src_group_name + OR ip_protocol, from_port, to_port, + and cidr_ip. In other words, either you are authorizing another + group or you are authorizing some ip-based rule. + + :type ip_protocol: string + :param ip_protocol: Either tcp | udp | icmp + + :type from_port: int + :param from_port: The beginning port number you are enabling + + :type to_port: int + :param to_port: The ending port number you are enabling + + :type to_port: string + :param to_port: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :type src_group: :class:`boto.ec2.securitygroup.SecurityGroup` or + :class:`boto.ec2.securitygroup.GroupOrCIDR` + + :rtype: bool + :return: True if successful. + """ + if src_group: + cidr_ip = None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.authorize_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.add_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def revoke(self, ip_protocol=None, from_port=None, to_port=None, + cidr_ip=None, src_group=None): + if src_group: + cidr_ip=None + src_group_name = src_group.name + src_group_owner_id = src_group.owner_id + else: + src_group_name = None + src_group_owner_id = None + status = self.connection.revoke_security_group(self.name, + src_group_name, + src_group_owner_id, + ip_protocol, + from_port, + to_port, + cidr_ip) + if status: + self.remove_rule(ip_protocol, from_port, to_port, src_group_name, + src_group_owner_id, cidr_ip) + return status + + def copy_to_region(self, region, name=None): + """ + Create a copy of this security group in another region. + Note that the new security group will be a separate entity + and will not stay in sync automatically after the copy + operation. + + :type region: :class:`boto.ec2.regioninfo.RegionInfo` + :param region: The region to which this security group will be copied. + + :type name: string + :param name: The name of the copy. If not supplied, the copy + will have the same name as this security group. + + :rtype: :class:`boto.ec2.securitygroup.SecurityGroup` + :return: The new security group. + """ + if region.name == self.region: + raise BotoClientError('Unable to copy to the same Region') + conn_params = self.connection.get_params() + rconn = region.connect(**conn_params) + sg = rconn.create_security_group(name or self.name, self.description) + source_groups = [] + for rule in self.rules: + grant = rule.grants[0] + if grant.name: + if grant.name not in source_groups: + source_groups.append(grant.name) + sg.authorize(None, None, None, None, grant) + else: + sg.authorize(rule.ip_protocol, rule.from_port, rule.to_port, + grant.cidr_ip) + return sg + + def instances(self): + instances = [] + rs = self.connection.get_all_instances() + for reservation in rs: + uses_group = [g.id for g in reservation.groups if g.id == self.name] + if uses_group: + instances.extend(reservation.instances) + return instances + +class IPPermissions: + + def __init__(self, parent=None): + self.parent = parent + self.ip_protocol = None + self.from_port = None + self.to_port = None + self.grants = [] + + def __repr__(self): + return 'IPPermissions:%s(%s-%s)' % (self.ip_protocol, + self.from_port, self.to_port) + + def startElement(self, name, attrs, connection): + if name == 'item': + self.grants.append(GroupOrCIDR(self)) + return self.grants[-1] + return None + + def endElement(self, name, value, connection): + if name == 'ipProtocol': + self.ip_protocol = value + elif name == 'fromPort': + self.from_port = value + elif name == 'toPort': + self.to_port = value + else: + setattr(self, name, value) + + def add_grant(self, name=None, owner_id=None, cidr_ip=None): + grant = GroupOrCIDR(self) + grant.owner_id = owner_id + grant.name = name + grant.cidr_ip = cidr_ip + self.grants.append(grant) + return grant + +class GroupOrCIDR: + + def __init__(self, parent=None): + self.owner_id = None + self.name = None + self.cidr_ip = None + + def __repr__(self): + if self.cidr_ip: + return '%s' % self.cidr_ip + else: + return '%s-%s' % (self.name, self.owner_id) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'userId': + self.owner_id = value + elif name == 'groupName': + self.name = value + if name == 'cidrIp': + self.cidr_ip = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/ec2/snapshot.py b/backup/src/boto/ec2/snapshot.py new file mode 100644 index 0000000..bbe8ad4 --- /dev/null +++ b/backup/src/boto/ec2/snapshot.py @@ -0,0 +1,140 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic IP Snapshot +""" +from boto.ec2.ec2object import TaggedEC2Object + +class Snapshot(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.volume_id = None + self.status = None + self.progress = None + self.start_time = None + self.owner_id = None + self.volume_size = None + self.description = None + + def __repr__(self): + return 'Snapshot:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'snapshotId': + self.id = value + elif name == 'volumeId': + self.volume_id = value + elif name == 'status': + self.status = value + elif name == 'startTime': + self.start_time = value + elif name == 'ownerId': + self.owner_id = value + elif name == 'volumeSize': + try: + self.volume_size = int(value) + except: + self.volume_size = value + elif name == 'description': + self.description = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.progress = updated.progress + self.status = updated.status + + def update(self, validate=False): + """ + Update the data associated with this snapshot by querying EC2. + + :type validate: bool + :param validate: By default, if EC2 returns no data about the + snapshot the update method returns quietly. If + the validate param is True, however, it will + raise a ValueError exception if no data is + returned from EC2. + """ + rs = self.connection.get_all_snapshots([self.id]) + if len(rs) > 0: + self._update(rs[0]) + elif validate: + raise ValueError('%s is not a valid Snapshot ID' % self.id) + return self.progress + + def delete(self): + return self.connection.delete_snapshot(self.id) + + def get_permissions(self): + attrs = self.connection.get_snapshot_attribute(self.id, + attribute='createVolumePermission') + return attrs.attrs + + def share(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'add', + user_ids, + groups) + + def unshare(self, user_ids=None, groups=None): + return self.connection.modify_snapshot_attribute(self.id, + 'createVolumePermission', + 'remove', + user_ids, + groups) + + def reset_permissions(self): + return self.connection.reset_snapshot_attribute(self.id, 'createVolumePermission') + +class SnapshotAttribute: + + def __init__(self, parent=None): + self.snapshot_id = None + self.attrs = {} + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'createVolumePermission': + self.name = 'create_volume_permission' + elif name == 'group': + if self.attrs.has_key('groups'): + self.attrs['groups'].append(value) + else: + self.attrs['groups'] = [value] + elif name == 'userId': + if self.attrs.has_key('user_ids'): + self.attrs['user_ids'].append(value) + else: + self.attrs['user_ids'] = [value] + elif name == 'snapshotId': + self.snapshot_id = value + else: + setattr(self, name, value) + + + diff --git a/backup/src/boto/ec2/spotdatafeedsubscription.py b/backup/src/boto/ec2/spotdatafeedsubscription.py new file mode 100644 index 0000000..9b820a3 --- /dev/null +++ b/backup/src/boto/ec2/spotdatafeedsubscription.py @@ -0,0 +1,63 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Datafeed Subscription +""" +from boto.ec2.ec2object import EC2Object +from boto.ec2.spotinstancerequest import SpotInstanceStateFault + +class SpotDatafeedSubscription(EC2Object): + + def __init__(self, connection=None, owner_id=None, + bucket=None, prefix=None, state=None,fault=None): + EC2Object.__init__(self, connection) + self.owner_id = owner_id + self.bucket = bucket + self.prefix = prefix + self.state = state + self.fault = fault + + def __repr__(self): + return 'SpotDatafeedSubscription:%s' % self.bucket + + def startElement(self, name, attrs, connection): + if name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'ownerId': + self.owner_id = value + elif name == 'bucket': + self.bucket = value + elif name == 'prefix': + self.prefix = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_spot_datafeed_subscription() + diff --git a/backup/src/boto/ec2/spotinstancerequest.py b/backup/src/boto/ec2/spotinstancerequest.py new file mode 100644 index 0000000..06acb0f --- /dev/null +++ b/backup/src/boto/ec2/spotinstancerequest.py @@ -0,0 +1,113 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import TaggedEC2Object +from boto.ec2.launchspecification import LaunchSpecification + +class SpotInstanceStateFault(object): + + def __init__(self, code=None, message=None): + self.code = code + self.message = message + + def __repr__(self): + return '(%s, %s)' % (self.code, self.message) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'code': + self.code = value + elif name == 'message': + self.message = value + setattr(self, name, value) + +class SpotInstanceRequest(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.price = None + self.type = None + self.state = None + self.fault = None + self.valid_from = None + self.valid_until = None + self.launch_group = None + self.product_description = None + self.availability_zone_group = None + self.create_time = None + self.launch_specification = None + self.instance_id = None + + def __repr__(self): + return 'SpotInstanceRequest:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'launchSpecification': + self.launch_specification = LaunchSpecification(connection) + return self.launch_specification + elif name == 'fault': + self.fault = SpotInstanceStateFault() + return self.fault + else: + return None + + def endElement(self, name, value, connection): + if name == 'spotInstanceRequestId': + self.id = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'productDescription': + self.product_description = value + elif name == 'validFrom': + self.valid_from = value + elif name == 'validUntil': + self.valid_until = value + elif name == 'launchGroup': + self.launch_group = value + elif name == 'availabilityZoneGroup': + self.availability_zone_group = value + elif name == 'createTime': + self.create_time = value + elif name == 'instanceId': + self.instance_id = value + else: + setattr(self, name, value) + + def cancel(self): + self.connection.cancel_spot_instance_requests([self.id]) + + + diff --git a/backup/src/boto/ec2/spotpricehistory.py b/backup/src/boto/ec2/spotpricehistory.py new file mode 100644 index 0000000..d4e1711 --- /dev/null +++ b/backup/src/boto/ec2/spotpricehistory.py @@ -0,0 +1,52 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Spot Instance Request +""" + +from boto.ec2.ec2object import EC2Object + +class SpotPriceHistory(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.price = 0.0 + self.instance_type = None + self.product_description = None + self.timestamp = None + + def __repr__(self): + return 'SpotPriceHistory(%s):%2f' % (self.instance_type, self.price) + + def endElement(self, name, value, connection): + if name == 'instanceType': + self.instance_type = value + elif name == 'spotPrice': + self.price = float(value) + elif name == 'productDescription': + self.product_description = value + elif name == 'timestamp': + self.timestamp = value + else: + setattr(self, name, value) + + diff --git a/backup/src/boto/ec2/tag.py b/backup/src/boto/ec2/tag.py new file mode 100644 index 0000000..8032e6f --- /dev/null +++ b/backup/src/boto/ec2/tag.py @@ -0,0 +1,87 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class TagSet(dict): + """ + A TagSet is used to collect the tags associated with a particular + EC2 resource. Not all resources can be tagged but for those that + can, this dict object will be used to collect those values. See + :class:`boto.ec2.ec2object.TaggedEC2Object` for more details. + """ + + def __init__(self, connection=None): + self.connection = connection + self._current_key = None + self._current_value = None + + def startElement(self, name, attrs, connection): + if name == 'item': + self._current_key = None + self._current_value = None + return None + + def endElement(self, name, value, connection): + if name == 'key': + self._current_key = value + elif name == 'value': + self._current_value = value + elif name == 'item': + self[self._current_key] = self._current_value + + +class Tag(object): + """ + A Tag is used when creating or listing all tags related to + an AWS account. It records not only the key and value but + also the ID of the resource to which the tag is attached + as well as the type of the resource. + """ + + def __init__(self, connection=None, res_id=None, res_type=None, + name=None, value=None): + self.connection = connection + self.res_id = res_id + self.res_type = res_type + self.name = name + self.value = value + + def __repr__(self): + return 'Tag:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'resourceId': + self.res_id = value + elif name == 'resourceType': + self.res_type = value + elif name == 'key': + self.name = value + elif name == 'value': + self.value = value + else: + setattr(self, name, value) + + + + diff --git a/backup/src/boto/ec2/volume.py b/backup/src/boto/ec2/volume.py new file mode 100644 index 0000000..45345fa --- /dev/null +++ b/backup/src/boto/ec2/volume.py @@ -0,0 +1,227 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Elastic Block Storage Volume +""" +from boto.ec2.ec2object import TaggedEC2Object + +class Volume(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.create_time = None + self.status = None + self.size = None + self.snapshot_id = None + self.attach_data = None + self.zone = None + + def __repr__(self): + return 'Volume:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'attachmentSet': + self.attach_data = AttachmentSet() + return self.attach_data + elif name == 'tagSet': + self.tags = boto.resultset.ResultSet([('item', Tag)]) + return self.tags + else: + return None + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'createTime': + self.create_time = value + elif name == 'status': + if value != '': + self.status = value + elif name == 'size': + self.size = int(value) + elif name == 'snapshotId': + self.snapshot_id = value + elif name == 'availabilityZone': + self.zone = value + else: + setattr(self, name, value) + + def _update(self, updated): + self.__dict__.update(updated.__dict__) + + def update(self, validate=False): + """ + Update the data associated with this volume by querying EC2. + + :type validate: bool + :param validate: By default, if EC2 returns no data about the + volume the update method returns quietly. If + the validate param is True, however, it will + raise a ValueError exception if no data is + returned from EC2. + """ + rs = self.connection.get_all_volumes([self.id]) + if len(rs) > 0: + self._update(rs[0]) + elif validate: + raise ValueError('%s is not a valid Volume ID' % self.id) + return self.status + + def delete(self): + """ + Delete this EBS volume. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_volume(self.id) + + def attach(self, instance_id, device): + """ + Attach this EBS volume to an EC2 instance. + + :type instance_id: str + :param instance_id: The ID of the EC2 instance to which it will + be attached. + + :type device: str + :param device: The device on the instance through which the + volume will be exposted (e.g. /dev/sdh) + + :rtype: bool + :return: True if successful + """ + return self.connection.attach_volume(self.id, instance_id, device) + + def detach(self, force=False): + """ + Detach this EBS volume from an EC2 instance. + + :type force: bool + :param force: Forces detachment if the previous detachment attempt did + not occur cleanly. This option can lead to data loss or + a corrupted file system. Use this option only as a last + resort to detach a volume from a failed instance. The + instance will not have an opportunity to flush file system + caches nor file system meta data. If you use this option, + you must perform file system check and repair procedures. + + :rtype: bool + :return: True if successful + """ + instance_id = None + if self.attach_data: + instance_id = self.attach_data.instance_id + device = None + if self.attach_data: + device = self.attach_data.device + return self.connection.detach_volume(self.id, instance_id, device, force) + + def create_snapshot(self, description=None): + """ + Create a snapshot of this EBS Volume. + + :type description: str + :param description: A description of the snapshot. Limited to 256 characters. + + :rtype: bool + :return: True if successful + """ + return self.connection.create_snapshot(self.id, description) + + def volume_state(self): + """ + Returns the state of the volume. Same value as the status attribute. + """ + return self.status + + def attachment_state(self): + """ + Get the attachment state. + """ + state = None + if self.attach_data: + state = self.attach_data.status + return state + + def snapshots(self, owner=None, restorable_by=None): + """ + Get all snapshots related to this volume. Note that this requires + that all available snapshots for the account be retrieved from EC2 + first and then the list is filtered client-side to contain only + those for this volume. + + :type owner: str + :param owner: If present, only the snapshots owned by the specified user + will be returned. Valid values are: + self | amazon | AWS Account ID + + :type restorable_by: str + :param restorable_by: If present, only the snapshots that are restorable + by the specified account id will be returned. + + :rtype: list of L{boto.ec2.snapshot.Snapshot} + :return: The requested Snapshot objects + + """ + rs = self.connection.get_all_snapshots(owner=owner, + restorable_by=restorable_by) + mine = [] + for snap in rs: + if snap.volume_id == self.id: + mine.append(snap) + return mine + +class AttachmentSet(object): + + def __init__(self): + self.id = None + self.instance_id = None + self.status = None + self.attach_time = None + self.device = None + + def __repr__(self): + return 'AttachmentSet:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'volumeId': + self.id = value + elif name == 'instanceId': + self.instance_id = value + elif name == 'status': + self.status = value + elif name == 'attachTime': + self.attach_time = value + elif name == 'device': + self.device = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/ec2/zone.py b/backup/src/boto/ec2/zone.py new file mode 100644 index 0000000..aec79b2 --- /dev/null +++ b/backup/src/boto/ec2/zone.py @@ -0,0 +1,47 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an EC2 Availability Zone +""" +from boto.ec2.ec2object import EC2Object + +class Zone(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.name = None + self.state = None + + def __repr__(self): + return 'Zone:%s' % self.name + + def endElement(self, name, value, connection): + if name == 'zoneName': + self.name = value + elif name == 'zoneState': + self.state = value + else: + setattr(self, name, value) + + + + diff --git a/backup/src/boto/ecs/__init__.py b/backup/src/boto/ecs/__init__.py new file mode 100644 index 0000000..db86dd5 --- /dev/null +++ b/backup/src/boto/ecs/__init__.py @@ -0,0 +1,84 @@ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.connection import AWSQueryConnection, AWSAuthConnection +import time +import urllib +import xml.sax +from boto.ecs.item import ItemSet +from boto import handler + +class ECSConnection(AWSQueryConnection): + """ECommerse Connection""" + + APIVersion = '2010-11-01' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='ecs.amazonaws.com', + debug=0, https_connection_factory=None, path='/'): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def _required_auth_capability(self): + return ['ecs'] + + def get_response(self, action, params, page=0, itemSet=None): + """ + Utility method to handle calls to ECS and parsing of responses. + """ + params['Service'] = "AWSECommerceService" + params['Operation'] = action + if page: + params['ItemPage'] = page + response = self.make_request(None, params, "/onca/xml") + body = response.read() + boto.log.debug(body) + + if response.status != 200: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + if itemSet == None: + rs = ItemSet(self, action, params, page) + else: + rs = itemSet + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + # + # Group methods + # + + def item_search(self, search_index, **params): + """ + Returns items that satisfy the search criteria, including one or more search + indices. + + For a full list of search terms, + :see: http://docs.amazonwebservices.com/AWSECommerceService/2010-09-01/DG/index.html?ItemSearch.html + """ + params['SearchIndex'] = search_index + return self.get_response('ItemSearch', params) diff --git a/backup/src/boto/ecs/item.py b/backup/src/boto/ecs/item.py new file mode 100644 index 0000000..29588b8 --- /dev/null +++ b/backup/src/boto/ecs/item.py @@ -0,0 +1,153 @@ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +import xml.sax +import cgi +from StringIO import StringIO + +class ResponseGroup(xml.sax.ContentHandler): + """A Generic "Response Group", which can + be anything from the entire list of Items to + specific response elements within an item""" + + def __init__(self, connection=None, nodename=None): + """Initialize this Item""" + self._connection = connection + self._nodename = nodename + self._nodepath = [] + self._curobj = None + self._xml = StringIO() + + def __repr__(self): + return '<%s: %s>' % (self.__class__.__name__, self.__dict__) + + # + # Attribute Functions + # + def get(self, name): + return self.__dict__.get(name) + + def set(self, name, value): + self.__dict__[name] = value + + def to_xml(self): + return "<%s>%s" % (self._nodename, self._xml.getvalue(), self._nodename) + + # + # XML Parser functions + # + def startElement(self, name, attrs, connection): + self._xml.write("<%s>" % name) + self._nodepath.append(name) + if len(self._nodepath) == 1: + obj = ResponseGroup(self._connection) + self.set(name, obj) + self._curobj = obj + elif self._curobj: + self._curobj.startElement(name, attrs, connection) + return None + + def endElement(self, name, value, connection): + self._xml.write("%s" % (cgi.escape(value).replace("&amp;", "&"), name)) + if len(self._nodepath) == 0: + return + obj = None + curval = self.get(name) + if len(self._nodepath) == 1: + if value or not curval: + self.set(name, value) + if self._curobj: + self._curobj = None + #elif len(self._nodepath) == 2: + #self._curobj = None + elif self._curobj: + self._curobj.endElement(name, value, connection) + self._nodepath.pop() + return None + + +class Item(ResponseGroup): + """A single Item""" + + def __init__(self, connection=None): + """Initialize this Item""" + ResponseGroup.__init__(self, connection, "Item") + +class ItemSet(ResponseGroup): + """A special ResponseGroup that has built-in paging, and + only creates new Items on the "Item" tag""" + + def __init__(self, connection, action, params, page=0): + ResponseGroup.__init__(self, connection, "Items") + self.objs = [] + self.iter = None + self.page = page + self.action = action + self.params = params + self.curItem = None + self.total_results = 0 + self.total_pages = 0 + + def startElement(self, name, attrs, connection): + if name == "Item": + self.curItem = Item(self._connection) + elif self.curItem != None: + self.curItem.startElement(name, attrs, connection) + return None + + def endElement(self, name, value, connection): + if name == 'TotalResults': + self.total_results = value + elif name == 'TotalPages': + self.total_pages = value + elif name == "Item": + self.objs.append(self.curItem) + self._xml.write(self.curItem.to_xml()) + self.curItem = None + elif self.curItem != None: + self.curItem.endElement(name, value, connection) + return None + + def next(self): + """Special paging functionality""" + if self.iter == None: + self.iter = iter(self.objs) + try: + return self.iter.next() + except StopIteration: + self.iter = None + self.objs = [] + if int(self.page) < int(self.total_pages): + self.page += 1 + self._connection.get_response(self.action, self.params, self.page, self) + return self.next() + else: + raise + + def __iter__(self): + return self + + def to_xml(self): + """Override to first fetch everything""" + for item in self: + pass + return ResponseGroup.to_xml(self) diff --git a/backup/src/boto/emr/__init__.py b/backup/src/boto/emr/__init__.py new file mode 100644 index 0000000..3c33f9a --- /dev/null +++ b/backup/src/boto/emr/__init__.py @@ -0,0 +1,30 @@ +# Copyright (c) 2010 Spotify AB +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module provies an interface to the Elastic MapReduce (EMR) +service from AWS. +""" +from connection import EmrConnection +from step import Step, StreamingStep, JarStep +from bootstrap_action import BootstrapAction + + diff --git a/backup/src/boto/emr/bootstrap_action.py b/backup/src/boto/emr/bootstrap_action.py new file mode 100644 index 0000000..c1c9038 --- /dev/null +++ b/backup/src/boto/emr/bootstrap_action.py @@ -0,0 +1,43 @@ +# Copyright (c) 2010 Spotify AB +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class BootstrapAction(object): + def __init__(self, name, path, bootstrap_action_args): + self.name = name + self.path = path + + if isinstance(bootstrap_action_args, basestring): + bootstrap_action_args = [bootstrap_action_args] + + self.bootstrap_action_args = bootstrap_action_args + + def args(self): + args = [] + + if self.bootstrap_action_args: + args.extend(self.bootstrap_action_args) + + return args + + def __repr__(self): + return '%s.%s(name=%r, path=%r, bootstrap_action_args=%r)' % ( + self.__class__.__module__, self.__class__.__name__, + self.name, self.path, self.bootstrap_action_args) diff --git a/backup/src/boto/emr/connection.py b/backup/src/boto/emr/connection.py new file mode 100644 index 0000000..f0145e3 --- /dev/null +++ b/backup/src/boto/emr/connection.py @@ -0,0 +1,280 @@ +# Copyright (c) 2010 Spotify AB +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EMR service +""" +import types + +import boto +from boto.ec2.regioninfo import RegionInfo +from boto.emr.emrobject import JobFlow, RunJobFlowResponse +from boto.emr.step import JarStep +from boto.connection import AWSQueryConnection +from boto.exception import EmrResponseError + +class EmrConnection(AWSQueryConnection): + + APIVersion = boto.config.get('Boto', 'emr_version', '2009-03-31') + DefaultRegionName = boto.config.get('Boto', 'emr_region_name', 'us-east-1') + DefaultRegionEndpoint = boto.config.get('Boto', 'emr_region_endpoint', + 'elasticmapreduce.amazonaws.com') + ResponseError = EmrResponseError + + # Constants for AWS Console debugging + DebuggingJar = 's3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar' + DebuggingArgs = 's3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + if not region: + region = RegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['emr'] + + def describe_jobflow(self, jobflow_id): + """ + Describes a single Elastic MapReduce job flow + + :type jobflow_id: str + :param jobflow_id: The job flow id of interest + """ + jobflows = self.describe_jobflows(jobflow_ids=[jobflow_id]) + if jobflows: + return jobflows[0] + + def describe_jobflows(self, states=None, jobflow_ids=None, + created_after=None, created_before=None): + """ + Retrieve all the Elastic MapReduce job flows on your account + + :type states: list + :param states: A list of strings with job flow states wanted + + :type jobflow_ids: list + :param jobflow_ids: A list of job flow IDs + :type created_after: datetime + :param created_after: Bound on job flow creation time + + :type created_before: datetime + :param created_before: Bound on job flow creation time + """ + params = {} + + if states: + self.build_list_params(params, states, 'JobFlowStates.member') + if jobflow_ids: + self.build_list_params(params, jobflow_ids, 'JobFlowIds.member') + if created_after: + params['CreatedAfter'] = created_after.strftime('%Y-%m-%dT%H:%M:%S') + if created_before: + params['CreatedBefore'] = created_before.strftime('%Y-%m-%dT%H:%M:%S') + + return self.get_list('DescribeJobFlows', params, [('member', JobFlow)]) + + def terminate_jobflow(self, jobflow_id): + """ + Terminate an Elastic MapReduce job flow + + :type jobflow_id: str + :param jobflow_id: A jobflow id + """ + self.terminate_jobflows([jobflow_id]) + + def terminate_jobflows(self, jobflow_ids): + """ + Terminate an Elastic MapReduce job flow + + :type jobflow_ids: list + :param jobflow_ids: A list of job flow IDs + """ + params = {} + self.build_list_params(params, jobflow_ids, 'JobFlowIds.member') + return self.get_status('TerminateJobFlows', params) + + def add_jobflow_steps(self, jobflow_id, steps): + """ + Adds steps to a jobflow + + :type jobflow_id: str + :param jobflow_id: The job flow id + :type steps: list(boto.emr.Step) + :param steps: A list of steps to add to the job + """ + if type(steps) != types.ListType: + steps = [steps] + params = {} + params['JobFlowId'] = jobflow_id + + # Step args + step_args = [self._build_step_args(step) for step in steps] + params.update(self._build_step_list(step_args)) + + return self.get_object('AddJobFlowSteps', params, RunJobFlowResponse) + + def run_jobflow(self, name, log_uri, ec2_keyname=None, availability_zone=None, + master_instance_type='m1.small', + slave_instance_type='m1.small', num_instances=1, + action_on_failure='TERMINATE_JOB_FLOW', keep_alive=False, + enable_debugging=False, + hadoop_version='0.18', + steps=[], + bootstrap_actions=[]): + """ + Runs a job flow + + :type name: str + :param name: Name of the job flow + :type log_uri: str + :param log_uri: URI of the S3 bucket to place logs + :type ec2_keyname: str + :param ec2_keyname: EC2 key used for the instances + :type availability_zone: str + :param availability_zone: EC2 availability zone of the cluster + :type master_instance_type: str + :param master_instance_type: EC2 instance type of the master + :type slave_instance_type: str + :param slave_instance_type: EC2 instance type of the slave nodes + :type num_instances: int + :param num_instances: Number of instances in the Hadoop cluster + :type action_on_failure: str + :param action_on_failure: Action to take if a step terminates + :type keep_alive: bool + :param keep_alive: Denotes whether the cluster should stay alive upon completion + :type enable_debugging: bool + :param enable_debugging: Denotes whether AWS console debugging should be enabled. + :type steps: list(boto.emr.Step) + :param steps: List of steps to add with the job + + :rtype: str + :return: The jobflow id + """ + params = {} + if action_on_failure: + params['ActionOnFailure'] = action_on_failure + params['Name'] = name + params['LogUri'] = log_uri + + # Instance args + instance_params = self._build_instance_args(ec2_keyname, availability_zone, + master_instance_type, slave_instance_type, + num_instances, keep_alive, hadoop_version) + params.update(instance_params) + + # Debugging step from EMR API docs + if enable_debugging: + debugging_step = JarStep(name='Setup Hadoop Debugging', + action_on_failure='TERMINATE_JOB_FLOW', + main_class=None, + jar=self.DebuggingJar, + step_args=self.DebuggingArgs) + steps.insert(0, debugging_step) + + # Step args + if steps: + step_args = [self._build_step_args(step) for step in steps] + params.update(self._build_step_list(step_args)) + + if bootstrap_actions: + bootstrap_action_args = [self._build_bootstrap_action_args(bootstrap_action) for bootstrap_action in bootstrap_actions] + params.update(self._build_bootstrap_action_list(bootstrap_action_args)) + + response = self.get_object('RunJobFlow', params, RunJobFlowResponse) + return response.jobflowid + + def _build_bootstrap_action_args(self, bootstrap_action): + bootstrap_action_params = {} + bootstrap_action_params['ScriptBootstrapAction.Path'] = bootstrap_action.path + + try: + bootstrap_action_params['Name'] = bootstrap_action.name + except AttributeError: + pass + + args = bootstrap_action.args() + if args: + self.build_list_params(bootstrap_action_params, args, 'ScriptBootstrapAction.Args.member') + + return bootstrap_action_params + + def _build_step_args(self, step): + step_params = {} + step_params['ActionOnFailure'] = step.action_on_failure + step_params['HadoopJarStep.Jar'] = step.jar() + + main_class = step.main_class() + if main_class: + step_params['HadoopJarStep.MainClass'] = main_class + + args = step.args() + if args: + self.build_list_params(step_params, args, 'HadoopJarStep.Args.member') + + step_params['Name'] = step.name + return step_params + + def _build_bootstrap_action_list(self, bootstrap_actions): + if type(bootstrap_actions) != types.ListType: + bootstrap_actions = [bootstrap_actions] + + params = {} + for i, bootstrap_action in enumerate(bootstrap_actions): + for key, value in bootstrap_action.iteritems(): + params['BootstrapActions.member.%s.%s' % (i + 1, key)] = value + return params + + def _build_step_list(self, steps): + if type(steps) != types.ListType: + steps = [steps] + + params = {} + for i, step in enumerate(steps): + for key, value in step.iteritems(): + params['Steps.member.%s.%s' % (i+1, key)] = value + return params + + def _build_instance_args(self, ec2_keyname, availability_zone, master_instance_type, + slave_instance_type, num_instances, keep_alive, hadoop_version): + params = { + 'Instances.MasterInstanceType' : master_instance_type, + 'Instances.SlaveInstanceType' : slave_instance_type, + 'Instances.InstanceCount' : num_instances, + 'Instances.KeepJobFlowAliveWhenNoSteps' : str(keep_alive).lower(), + 'Instances.HadoopVersion' : hadoop_version + } + + if ec2_keyname: + params['Instances.Ec2KeyName'] = ec2_keyname + if availability_zone: + params['Placement'] = availability_zone + + return params + diff --git a/backup/src/boto/emr/emrobject.py b/backup/src/boto/emr/emrobject.py new file mode 100644 index 0000000..0ffe292 --- /dev/null +++ b/backup/src/boto/emr/emrobject.py @@ -0,0 +1,141 @@ +# Copyright (c) 2010 Spotify AB +# Copyright (c) 2010 Jeremy Thurgood +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +This module contains EMR response objects +""" + +from boto.resultset import ResultSet + + +class EmrObject(object): + Fields = set() + + def __init__(self, connection=None): + self.connection = connection + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in self.Fields: + setattr(self, name.lower(), value) + + +class RunJobFlowResponse(EmrObject): + Fields = set(['JobFlowId']) + + +class Arg(EmrObject): + def __init__(self, connection=None): + self.value = None + + def endElement(self, name, value, connection): + self.value = value + + +class BootstrapAction(EmrObject): + Fields = set(['Name', + 'Args', + 'Path']) + + +class Step(EmrObject): + Fields = set(['Name', + 'ActionOnFailure', + 'CreationDateTime', + 'StartDateTime', + 'EndDateTime', + 'LastStateChangeReason', + 'State']) + + def __init__(self, connection=None): + self.connection = connection + self.args = None + + def startElement(self, name, attrs, connection): + if name == 'Args': + self.args = ResultSet([('member', Arg)]) + return self.args + + +class InstanceGroup(EmrObject): + Fields = set(['Name', + 'CreationDateTime', + 'InstanceRunningCount', + 'StartDateTime', + 'ReadyDateTime', + 'State', + 'EndDateTime', + 'InstanceRequestCount', + 'InstanceType', + 'Market', + 'LastStateChangeReason', + 'InstanceRole', + 'InstanceGroupId', + 'LaunchGroup', + 'SpotPrice']) + + +class JobFlow(EmrObject): + Fields = set(['CreationDateTime', + 'StartDateTime', + 'State', + 'EndDateTime', + 'Id', + 'InstanceCount', + 'JobFlowId', + 'LogUri', + 'MasterPublicDnsName', + 'MasterInstanceId', + 'Name', + 'Placement', + 'RequestId', + 'Type', + 'Value', + 'AvailabilityZone', + 'SlaveInstanceType', + 'MasterInstanceType', + 'Ec2KeyName', + 'InstanceCount', + 'KeepJobFlowAliveWhenNoSteps', + 'LastStateChangeReason']) + + def __init__(self, connection=None): + self.connection = connection + self.steps = None + self.instancegroups = None + self.bootstrapactions = None + + def startElement(self, name, attrs, connection): + if name == 'Steps': + self.steps = ResultSet([('member', Step)]) + return self.steps + elif name == 'InstanceGroups': + self.instancegroups = ResultSet([('member', InstanceGroup)]) + return self.instancegroups + elif name == 'BootstrapActions': + self.bootstrapactions = ResultSet([('member', BootstrapAction)]) + return self.bootstrapactions + else: + return None + diff --git a/backup/src/boto/emr/step.py b/backup/src/boto/emr/step.py new file mode 100644 index 0000000..a444261 --- /dev/null +++ b/backup/src/boto/emr/step.py @@ -0,0 +1,179 @@ +# Copyright (c) 2010 Spotify AB +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Step(object): + """ + Jobflow Step base class + """ + def jar(self): + """ + :rtype: str + :return: URI to the jar + """ + raise NotImplemented() + + def args(self): + """ + :rtype: list(str) + :return: List of arguments for the step + """ + raise NotImplemented() + + def main_class(self): + """ + :rtype: str + :return: The main class name + """ + raise NotImplemented() + + +class JarStep(Step): + """ + Custom jar step + """ + def __init__(self, name, jar, main_class=None, + action_on_failure='TERMINATE_JOB_FLOW', step_args=None): + """ + A elastic mapreduce step that executes a jar + + :type name: str + :param name: The name of the step + :type jar: str + :param jar: S3 URI to the Jar file + :type main_class: str + :param main_class: The class to execute in the jar + :type action_on_failure: str + :param action_on_failure: An action, defined in the EMR docs to take on failure. + :type step_args: list(str) + :param step_args: A list of arguments to pass to the step + """ + self.name = name + self._jar = jar + self._main_class = main_class + self.action_on_failure = action_on_failure + + if isinstance(step_args, basestring): + step_args = [step_args] + + self.step_args = step_args + + def jar(self): + return self._jar + + def args(self): + args = [] + + if self.step_args: + args.extend(self.step_args) + + return args + + def main_class(self): + return self._main_class + + +class StreamingStep(Step): + """ + Hadoop streaming step + """ + def __init__(self, name, mapper, reducer=None, + action_on_failure='TERMINATE_JOB_FLOW', + cache_files=None, cache_archives=None, + step_args=None, input=None, output=None): + """ + A hadoop streaming elastic mapreduce step + + :type name: str + :param name: The name of the step + :type mapper: str + :param mapper: The mapper URI + :type reducer: str + :param reducer: The reducer URI + :type action_on_failure: str + :param action_on_failure: An action, defined in the EMR docs to take on failure. + :type cache_files: list(str) + :param cache_files: A list of cache files to be bundled with the job + :type cache_archives: list(str) + :param cache_archives: A list of jar archives to be bundled with the job + :type step_args: list(str) + :param step_args: A list of arguments to pass to the step + :type input: str or a list of str + :param input: The input uri + :type output: str + :param output: The output uri + """ + self.name = name + self.mapper = mapper + self.reducer = reducer + self.action_on_failure = action_on_failure + self.cache_files = cache_files + self.cache_archives = cache_archives + self.input = input + self.output = output + + if isinstance(step_args, basestring): + step_args = [step_args] + + self.step_args = step_args + + def jar(self): + return '/home/hadoop/contrib/streaming/hadoop-0.18-streaming.jar' + + def main_class(self): + return None + + def args(self): + args = ['-mapper', self.mapper] + + if self.reducer: + args.extend(['-reducer', self.reducer]) + + if self.input: + if isinstance(self.input, list): + for input in self.input: + args.extend(('-input', input)) + else: + args.extend(('-input', self.input)) + if self.output: + args.extend(('-output', self.output)) + + if self.cache_files: + for cache_file in self.cache_files: + args.extend(('-cacheFile', cache_file)) + + if self.cache_archives: + for cache_archive in self.cache_archives: + args.extend(('-cacheArchive', cache_archive)) + + if self.step_args: + args.extend(self.step_args) + + if not self.reducer: + args.extend(['-jobconf', 'mapred.reduce.tasks=0']) + + return args + + def __repr__(self): + return '%s.%s(name=%r, mapper=%r, reducer=%r, action_on_failure=%r, cache_files=%r, cache_archives=%r, step_args=%r, input=%r, output=%r)' % ( + self.__class__.__module__, self.__class__.__name__, + self.name, self.mapper, self.reducer, self.action_on_failure, + self.cache_files, self.cache_archives, self.step_args, + self.input, self.output) diff --git a/backup/src/boto/emr/tests/test_emr_responses.py b/backup/src/boto/emr/tests/test_emr_responses.py new file mode 100644 index 0000000..77ec494 --- /dev/null +++ b/backup/src/boto/emr/tests/test_emr_responses.py @@ -0,0 +1,373 @@ +# Copyright (c) 2010 Jeremy Thurgood +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +# NOTE: These tests only cover the very simple cases I needed to test +# for the InstanceGroup fix. + +import xml.sax +import unittest + +from boto import handler +from boto.emr import emrobject +from boto.resultset import ResultSet + + +JOB_FLOW_EXAMPLE = """ + + + + + + 2009-01-28T21:49:16Z + 2009-01-28T21:49:16Z + STARTING + + MyJobFlowName + mybucket/subdir/ + + + + 2009-01-28T21:49:16Z + PENDING + + + + MyJarFile + MyMailClass + + arg1 + arg2 + + + + MyStepName + CONTINUE + + + + j-3UN6WX5RRO2AG + + + us-east-1a + + m1.small + m1.small + myec2keyname + 4 + true + + + + + + 9cea3229-ed85-11dd-9877-6fad448a8419 + + +""" + +JOB_FLOW_COMPLETED = """ + + + + + + 2010-10-21T01:00:25Z + Steps completed + 2010-10-21T01:03:59Z + 2010-10-21T01:03:59Z + COMPLETED + 2010-10-21T01:44:18Z + + + RealJobFlowName + s3n://example.emrtest.scripts/jobflow_logs/ + + + + + s3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar + + s3n://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch + + + + Setup Hadoop Debugging + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:03:59Z + COMPLETED + 2010-10-21T01:04:22Z + + + + + + /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar + + -mapper + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialMapper.py + -reducer + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-InitialReducer.py + -input + s3://example.emrtest.data/raw/2010/10/20/* + -input + s3://example.emrtest.data/raw/2010/10/19/* + -input + s3://example.emrtest.data/raw/2010/10/18/* + -input + s3://example.emrtest.data/raw/2010/10/17/* + -input + s3://example.emrtest.data/raw/2010/10/16/* + -input + s3://example.emrtest.data/raw/2010/10/15/* + -input + s3://example.emrtest.data/raw/2010/10/14/* + -output + s3://example.emrtest.crunched/ + + + + testjob_Initial + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:04:22Z + COMPLETED + 2010-10-21T01:36:18Z + + + + + + /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar + + -mapper + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Mapper.py + -reducer + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step1Reducer.py + -input + s3://example.emrtest.crunched/* + -output + s3://example.emrtest.step1/ + + + + testjob_step1 + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:36:18Z + COMPLETED + 2010-10-21T01:37:51Z + + + + + + /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar + + -mapper + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Mapper.py + -reducer + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step2Reducer.py + -input + s3://example.emrtest.crunched/* + -output + s3://example.emrtest.step2/ + + + + testjob_step2 + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:37:51Z + COMPLETED + 2010-10-21T01:39:32Z + + + + + + /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar + + -mapper + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Mapper.py + -reducer + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step3Reducer.py + -input + s3://example.emrtest.step1/* + -output + s3://example.emrtest.step3/ + + + + testjob_step3 + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:39:32Z + COMPLETED + 2010-10-21T01:41:22Z + + + + + + /home/hadoop/contrib/streaming/hadoop-0.20-streaming.jar + + -mapper + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Mapper.py + -reducer + s3://example.emrtest.scripts/81d8-5a9d3df4a86c-step4Reducer.py + -input + s3://example.emrtest.step1/* + -output + s3://example.emrtest.step4/ + + + + testjob_step4 + TERMINATE_JOB_FLOW + + + 2010-10-21T01:00:25Z + 2010-10-21T01:41:22Z + COMPLETED + 2010-10-21T01:43:03Z + + + + j-3H3Q13JPFLU22 + + m1.large + i-64c21609 + + us-east-1b + + + + 2010-10-21T01:00:25Z + 0 + 2010-10-21T01:02:09Z + 2010-10-21T01:03:03Z + ENDED + 2010-10-21T01:44:18Z + 1 + m1.large + ON_DEMAND + Job flow terminated + MASTER + ig-EVMHOZJ2SCO8 + master + + + 2010-10-21T01:00:25Z + 0 + 2010-10-21T01:03:59Z + 2010-10-21T01:03:59Z + ENDED + 2010-10-21T01:44:18Z + 9 + m1.large + ON_DEMAND + Job flow terminated + CORE + ig-YZHDYVITVHKB + slave + + + 40 + 0.20 + m1.large + ec2-184-72-153-139.compute-1.amazonaws.com + myubersecurekey + 10 + false + + + + + + c31e701d-dcb4-11df-b5d9-337fc7fe4773 + + +""" + + +class TestEMRResponses(unittest.TestCase): + def _parse_xml(self, body, markers): + rs = ResultSet(markers) + h = handler.XmlHandler(rs, None) + xml.sax.parseString(body, h) + return rs + + def _assert_fields(self, response, **fields): + for field, expected in fields.items(): + actual = getattr(response, field) + self.assertEquals(expected, actual, + "Field %s: %r != %r" % (field, expected, actual)) + + def test_JobFlows_example(self): + [jobflow] = self._parse_xml(JOB_FLOW_EXAMPLE, + [('member', emrobject.JobFlow)]) + self._assert_fields(jobflow, + creationdatetime='2009-01-28T21:49:16Z', + startdatetime='2009-01-28T21:49:16Z', + state='STARTING', + instancecount='4', + jobflowid='j-3UN6WX5RRO2AG', + loguri='mybucket/subdir/', + name='MyJobFlowName', + availabilityzone='us-east-1a', + slaveinstancetype='m1.small', + masterinstancetype='m1.small', + ec2keyname='myec2keyname', + keepjobflowalivewhennosteps='true') + + def test_JobFlows_completed(self): + [jobflow] = self._parse_xml(JOB_FLOW_COMPLETED, + [('member', emrobject.JobFlow)]) + self._assert_fields(jobflow, + creationdatetime='2010-10-21T01:00:25Z', + startdatetime='2010-10-21T01:03:59Z', + enddatetime='2010-10-21T01:44:18Z', + state='COMPLETED', + instancecount='10', + jobflowid='j-3H3Q13JPFLU22', + loguri='s3n://example.emrtest.scripts/jobflow_logs/', + name='RealJobFlowName', + availabilityzone='us-east-1b', + slaveinstancetype='m1.large', + masterinstancetype='m1.large', + ec2keyname='myubersecurekey', + keepjobflowalivewhennosteps='false') + self.assertEquals(6, len(jobflow.steps)) + self.assertEquals(2, len(jobflow.instancegroups)) + diff --git a/backup/src/boto/exception.py b/backup/src/boto/exception.py new file mode 100644 index 0000000..718be46 --- /dev/null +++ b/backup/src/boto/exception.py @@ -0,0 +1,430 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Exception classes - Subclassing allows you to check for specific errors +""" +import base64 +import xml.sax +from boto import handler +from boto.resultset import ResultSet + + +class BotoClientError(StandardError): + """ + General Boto Client error (error accessing AWS) + """ + + def __init__(self, reason): + StandardError.__init__(self) + self.reason = reason + + def __repr__(self): + return 'BotoClientError: %s' % self.reason + + def __str__(self): + return 'BotoClientError: %s' % self.reason + +class SDBPersistenceError(StandardError): + + pass + +class StoragePermissionsError(BotoClientError): + """ + Permissions error when accessing a bucket or key on a storage service. + """ + pass + +class S3PermissionsError(StoragePermissionsError): + """ + Permissions error when accessing a bucket or key on S3. + """ + pass + +class GSPermissionsError(StoragePermissionsError): + """ + Permissions error when accessing a bucket or key on GS. + """ + pass + +class BotoServerError(StandardError): + + def __init__(self, status, reason, body=None): + StandardError.__init__(self) + self.status = status + self.reason = reason + self.body = body or '' + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + + # Attempt to parse the error response. If body isn't present, + # then just ignore the error response. + if self.body: + try: + h = handler.XmlHandler(self, self) + xml.sax.parseString(self.body, h) + except xml.sax.SAXParseException, pe: + # Go ahead and clean up anything that may have + # managed to get into the error data so we + # don't get partial garbage. + print "Warning: failed to parse error message from AWS: %s" % pe + self._cleanupParsedProperties() + + def __getattr__(self, name): + if name == 'message': + return self.error_message + if name == 'code': + return self.error_code + raise AttributeError + + def __repr__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def __str__(self): + return '%s: %s %s\n%s' % (self.__class__.__name__, + self.status, self.reason, self.body) + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name in ('RequestId', 'RequestID'): + self.request_id = value + elif name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + elif name == 'BoxUsage': + self.box_usage = value + return None + + def _cleanupParsedProperties(self): + self.request_id = None + self.error_code = None + self.error_message = None + self.box_usage = None + +class ConsoleOutput: + + def __init__(self, parent=None): + self.parent = parent + self.instance_id = None + self.timestamp = None + self.comment = None + self.output = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'instanceId': + self.instance_id = value + elif name == 'output': + self.output = base64.b64decode(value) + else: + setattr(self, name, value) + +class StorageCreateError(BotoServerError): + """ + Error creating a bucket or key on a storage service. + """ + def __init__(self, status, reason, body=None): + self.bucket = None + BotoServerError.__init__(self, status, reason, body) + + def endElement(self, name, value, connection): + if name == 'BucketName': + self.bucket = value + else: + return BotoServerError.endElement(self, name, value, connection) + +class S3CreateError(StorageCreateError): + """ + Error creating a bucket or key on S3. + """ + pass + +class GSCreateError(StorageCreateError): + """ + Error creating a bucket or key on GS. + """ + pass + +class StorageCopyError(BotoServerError): + """ + Error copying a key on a storage service. + """ + pass + +class S3CopyError(StorageCopyError): + """ + Error copying a key on S3. + """ + pass + +class GSCopyError(StorageCopyError): + """ + Error copying a key on GS. + """ + pass + +class SQSError(BotoServerError): + """ + General Error on Simple Queue Service. + """ + def __init__(self, status, reason, body=None): + self.detail = None + self.type = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Detail': + self.detail = value + elif name == 'Type': + self.type = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('detail', 'type'): + setattr(self, p, None) + +class SQSDecodeError(BotoClientError): + """ + Error when decoding an SQS message. + """ + def __init__(self, reason, message): + BotoClientError.__init__(self, reason) + self.message = message + + def __repr__(self): + return 'SQSDecodeError: %s' % self.reason + + def __str__(self): + return 'SQSDecodeError: %s' % self.reason + +class StorageResponseError(BotoServerError): + """ + Error in response from a storage service. + """ + def __init__(self, status, reason, body=None): + self.resource = None + BotoServerError.__init__(self, status, reason, body) + + def startElement(self, name, attrs, connection): + return BotoServerError.startElement(self, name, attrs, connection) + + def endElement(self, name, value, connection): + if name == 'Resource': + self.resource = value + else: + return BotoServerError.endElement(self, name, value, connection) + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + for p in ('resource'): + setattr(self, p, None) + +class S3ResponseError(StorageResponseError): + """ + Error in response from S3. + """ + pass + +class GSResponseError(StorageResponseError): + """ + Error in response from GS. + """ + pass + +class EC2ResponseError(BotoServerError): + """ + Error in response from EC2. + """ + + def __init__(self, status, reason, body=None): + self.errors = None + self._errorResultSet = [] + BotoServerError.__init__(self, status, reason, body) + self.errors = [ (e.error_code, e.error_message) \ + for e in self._errorResultSet ] + if len(self.errors): + self.error_code, self.error_message = self.errors[0] + + def startElement(self, name, attrs, connection): + if name == 'Errors': + self._errorResultSet = ResultSet([('Error', _EC2Error)]) + return self._errorResultSet + else: + return None + + def endElement(self, name, value, connection): + if name == 'RequestID': + self.request_id = value + else: + return None # don't call subclass here + + def _cleanupParsedProperties(self): + BotoServerError._cleanupParsedProperties(self) + self._errorResultSet = [] + for p in ('errors'): + setattr(self, p, None) + +class EmrResponseError(BotoServerError): + """ + Error in response from EMR + """ + pass + +class _EC2Error: + + def __init__(self, connection=None): + self.connection = connection + self.error_code = None + self.error_message = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Code': + self.error_code = value + elif name == 'Message': + self.error_message = value + else: + return None + +class SDBResponseError(BotoServerError): + """ + Error in responses from SDB. + """ + pass + +class AWSConnectionError(BotoClientError): + """ + General error connecting to Amazon Web Services. + """ + pass + +class StorageDataError(BotoClientError): + """ + Error receiving data from a storage service. + """ + pass + +class S3DataError(StorageDataError): + """ + Error receiving data from S3. + """ + pass + +class GSDataError(StorageDataError): + """ + Error receiving data from GS. + """ + pass + +class FPSResponseError(BotoServerError): + pass + +class InvalidUriError(Exception): + """Exception raised when URI is invalid.""" + + def __init__(self, message): + Exception.__init__(self) + self.message = message + +class InvalidAclError(Exception): + """Exception raised when ACL XML is invalid.""" + + def __init__(self, message): + Exception.__init__(self) + self.message = message + +class NoAuthHandlerFound(Exception): + """Is raised when no auth handlers were found ready to authenticate.""" + pass + +class TooManyAuthHandlerReadyToAuthenticate(Exception): + """Is raised when there are more than one auth handler ready. + + In normal situation there should only be one auth handler that is ready to + authenticate. In case where more than one auth handler is ready to + authenticate, we raise this exception, to prevent unpredictable behavior + when multiple auth handlers can handle a particular case and the one chosen + depends on the order they were checked. + """ + pass + +# Enum class for resumable upload failure disposition. +class ResumableTransferDisposition(object): + # START_OVER means an attempt to resume an existing transfer failed, + # and a new resumable upload should be attempted (without delay). + START_OVER = 'START_OVER' + + # WAIT_BEFORE_RETRY means the resumable transfer failed but that it can + # be retried after a time delay. + WAIT_BEFORE_RETRY = 'WAIT_BEFORE_RETRY' + + # ABORT means the resumable transfer failed and that delaying/retrying + # within the current process will not help. + ABORT = 'ABORT' + +class ResumableUploadException(Exception): + """ + Exception raised for various resumable upload problems. + + self.disposition is of type ResumableTransferDisposition. + """ + + def __init__(self, message, disposition): + Exception.__init__(self) + self.message = message + self.disposition = disposition + + def __repr__(self): + return 'ResumableUploadException("%s", %s)' % ( + self.message, self.disposition) + +class ResumableDownloadException(Exception): + """ + Exception raised for various resumable download problems. + + self.disposition is of type ResumableTransferDisposition. + """ + + def __init__(self, message, disposition): + Exception.__init__(self) + self.message = message + self.disposition = disposition + + def __repr__(self): + return 'ResumableDownloadException("%s", %s)' % ( + self.message, self.disposition) diff --git a/backup/src/boto/file/README b/backup/src/boto/file/README new file mode 100644 index 0000000..af82455 --- /dev/null +++ b/backup/src/boto/file/README @@ -0,0 +1,49 @@ +Handling of file:// URIs: + +This directory contains code to map basic boto connection, bucket, and key +operations onto files in the local filesystem, in support of file:// +URI operations. + +Bucket storage operations cannot be mapped completely onto a file system +because of the different naming semantics in these types of systems: the +former have a flat name space of objects within each named bucket; the +latter have a hierarchical name space of files, and nothing corresponding to +the notion of a bucket. The mapping we selected was guided by the desire +to achieve meaningful semantics for a useful subset of operations that can +be implemented polymorphically across both types of systems. We considered +several possibilities for mapping path names to bucket + object name: + +1) bucket = the file system root or local directory (for absolute vs +relative file:// URIs, respectively) and object = remainder of path. +We discarded this choice because the get_all_keys() method doesn't make +sense under this approach: Enumerating all files under the root or current +directory could include more than the caller intended. For example, +StorageUri("file:///usr/bin/X11/vim").get_all_keys() would enumerate all +files in the file system. + +2) bucket is treated mostly as an anonymous placeholder, with the object +name holding the URI path (minus the "file://" part). Two sub-options, +for object enumeration (the get_all_keys() call): + a) disallow get_all_keys(). This isn't great, as then the caller must + know the URI type before deciding whether to make this call. + b) return the single key for which this "bucket" was defined. + Note that this option means the app cannot use this API for listing + contents of the file system. While that makes the API less generally + useful, it avoids the potentially dangerous/unintended consequences + noted in option (1) above. + +We selected 2b, resulting in a class hierarchy where StorageUri is an abstract +class, with FileStorageUri and BucketStorageUri subclasses. + +Some additional notes: + +BucketStorageUri and FileStorageUri each implement these methods: + - clone_replace_name() creates a same-type URI with a + different object name - which is useful for various enumeration cases + (e.g., implementing wildcarding in a command line utility). + - names_container() determines if the given URI names a container for + multiple objects/files - i.e., a bucket or directory. + - names_singleton() determines if the given URI names an individual object + or file. + - is_file_uri() and is_cloud_uri() determine if the given URI is a + FileStorageUri or BucketStorageUri, respectively diff --git a/backup/src/boto/file/__init__.py b/backup/src/boto/file/__init__.py new file mode 100644 index 0000000..0210b47 --- /dev/null +++ b/backup/src/boto/file/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto + +from connection import FileConnection as Connection +from key import Key +from bucket import Bucket + +__all__ = ['Connection', 'Key', 'Bucket'] diff --git a/backup/src/boto/file/bucket.py b/backup/src/boto/file/bucket.py new file mode 100644 index 0000000..7a1636b --- /dev/null +++ b/backup/src/boto/file/bucket.py @@ -0,0 +1,101 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# File representation of bucket, for use with "file://" URIs. + +import os +from key import Key +from boto.file.simpleresultset import SimpleResultSet +from boto.s3.bucketlistresultset import BucketListResultSet + +class Bucket(object): + def __init__(self, name, contained_key): + """Instantiate an anonymous file-based Bucket around a single key. + """ + self.name = name + self.contained_key = contained_key + + def __iter__(self): + return iter(BucketListResultSet(self)) + + def __str__(self): + return 'anonymous bucket for file://' + self.contained_key + + def delete_key(self, key_name, headers=None, + version_id=None, mfa_token=None): + """ + Deletes a key from the bucket. + + :type key_name: string + :param key_name: The key name to delete + + :type version_id: string + :param version_id: Unused in this subclass. + + :type mfa_token: tuple or list of strings + :param mfa_token: Unused in this subclass. + """ + os.remove(key_name) + + def get_all_keys(self, headers=None, **params): + """ + This method returns the single key around which this anonymous Bucket + was instantiated. + + :rtype: SimpleResultSet + :return: The result from file system listing the keys requested + + """ + key = Key(self.name, self.contained_key) + return SimpleResultSet([key]) + + def get_key(self, key_name, headers=None, version_id=None): + """ + Check to see if a particular key exists within the bucket. + Returns: An instance of a Key object or None + + :type key_name: string + :param key_name: The name of the key to retrieve + + :type version_id: string + :param version_id: Unused in this subclass. + + :rtype: :class:`boto.file.key.Key` + :returns: A Key object from this bucket. + """ + fp = open(key_name, 'rb') + return Key(self.name, key_name, fp) + + def new_key(self, key_name=None): + """ + Creates a new key + + :type key_name: string + :param key_name: The name of the key to create + + :rtype: :class:`boto.file.key.Key` + :returns: An instance of the newly created key object + """ + dir_name = os.path.dirname(key_name) + if dir_name and not os.path.exists(dir_name): + os.makedirs(dir_name) + fp = open(key_name, 'wb') + return Key(self.name, key_name, fp) diff --git a/backup/src/boto/file/connection.py b/backup/src/boto/file/connection.py new file mode 100644 index 0000000..f453f71 --- /dev/null +++ b/backup/src/boto/file/connection.py @@ -0,0 +1,33 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# File representation of connection, for use with "file://" URIs. + +from bucket import Bucket + +class FileConnection(object): + + def __init__(self, file_storage_uri): + # FileConnections are per-file storage URI. + self.file_storage_uri = file_storage_uri + + def get_bucket(self, bucket_name, validate=True, headers=None): + return Bucket(bucket_name, self.file_storage_uri.object_name) diff --git a/backup/src/boto/file/key.py b/backup/src/boto/file/key.py new file mode 100644 index 0000000..af801a5 --- /dev/null +++ b/backup/src/boto/file/key.py @@ -0,0 +1,123 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# File representation of key, for use with "file://" URIs. + +import os, shutil, StringIO + +class Key(object): + + def __init__(self, bucket, name, fp=None): + self.bucket = bucket + self.full_path = name + self.name = name + self.fp = fp + + def __str__(self): + return 'file://' + self.full_path + + def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False): + """ + Retrieves a file from a Key + + :type fp: file + :param fp: File pointer to put the data into + + :type headers: string + :param: ignored in this subclass. + + :type cb: function + :param cb: ignored in this subclass. + + :type cb: int + :param num_cb: ignored in this subclass. + """ + key_file = open(self.full_path, 'rb') + shutil.copyfileobj(key_file, fp) + + def set_contents_from_file(self, fp, headers=None, replace=True, cb=None, + num_cb=10, policy=None, md5=None): + """ + Store an object in a file using the name of the Key object as the + key in file URI and the contents of the file pointed to by 'fp' as the + contents. + + :type fp: file + :param fp: the file whose contents to upload + + :type headers: dict + :param headers: ignored in this subclass. + + :type replace: bool + :param replace: If this parameter is False, the method + will first check to see if an object exists in the + bucket with the same key. If it does, it won't + overwrite it. The default value is True which will + overwrite the object. + + :type cb: function + :param cb: ignored in this subclass. + + :type cb: int + :param num_cb: ignored in this subclass. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: ignored in this subclass. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum + of the file as the first element and the Base64-encoded + version of the plain checksum as the second element. + This is the same format returned by the compute_md5 method. + :param md5: ignored in this subclass. + """ + if not replace and os.path.exists(self.full_path): + return + key_file = open(self.full_path, 'wb') + shutil.copyfileobj(fp, key_file) + key_file.close() + + def get_contents_as_string(self, headers=None, cb=None, num_cb=10, + torrent=False): + """ + Retrieve file data from the Key, and return contents as a string. + + :type headers: dict + :param headers: ignored in this subclass. + + :type cb: function + :param cb: ignored in this subclass. + + :type cb: int + :param num_cb: ignored in this subclass. + + :type cb: int + :param num_cb: ignored in this subclass. + + :type torrent: bool + :param torrent: ignored in this subclass. + + :rtype: string + :returns: The contents of the file as a string + """ + + fp = StringIO.StringIO() + self.get_contents_to_file(fp) + return fp.getvalue() diff --git a/backup/src/boto/file/simpleresultset.py b/backup/src/boto/file/simpleresultset.py new file mode 100644 index 0000000..5f94dc1 --- /dev/null +++ b/backup/src/boto/file/simpleresultset.py @@ -0,0 +1,30 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class SimpleResultSet(list): + """ + ResultSet facade built from a simple list, rather than via XML parsing. + """ + + def __init__(self, input_list): + for x in input_list: + self.append(x) + self.is_truncated = False diff --git a/backup/src/boto/fps/__init__.py b/backup/src/boto/fps/__init__.py new file mode 100644 index 0000000..2f44483 --- /dev/null +++ b/backup/src/boto/fps/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2008, Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/backup/src/boto/fps/connection.py b/backup/src/boto/fps/connection.py new file mode 100644 index 0000000..3d7812e --- /dev/null +++ b/backup/src/boto/fps/connection.py @@ -0,0 +1,356 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# Copyringt (c) 2010 Jason R. Coombs http://www.jaraco.com/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import base64 +import hmac +import hashlib +import urllib +import xml.sax +import uuid +import boto +import boto.utils +from boto import handler +from boto.connection import AWSQueryConnection +from boto.resultset import ResultSet +from boto.exception import FPSResponseError + +class FPSConnection(AWSQueryConnection): + + APIVersion = '2007-01-08' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host='fps.sandbox.amazonaws.com', debug=0, + https_connection_factory=None, path="/"): + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, host, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['fps'] + + def install_payment_instruction(self, instruction, token_type="Unrestricted", transaction_id=None): + """ + InstallPaymentInstruction + instruction: The PaymentInstruction to send, for example: + + MyRole=='Caller' orSay 'Roles do not match'; + + token_type: Defaults to "Unrestricted" + transaction_id: Defaults to a new ID + """ + + if(transaction_id == None): + transaction_id = uuid.uuid4() + params = {} + params['PaymentInstruction'] = instruction + params['TokenType'] = token_type + params['CallerReference'] = transaction_id + response = self.make_request("InstallPaymentInstruction", params) + return response + + def install_caller_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a caller + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the caller token. + """ + response = self.install_payment_instruction("MyRole=='Caller';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + caller_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "caller_token", caller_token) + except(IOError): + boto.config.save_user_option("FPS", "caller_token", caller_token) + return caller_token + else: + raise FPSResponseError(response.status, response.reason, body) + + def install_recipient_instruction(self, token_type="Unrestricted", transaction_id=None): + """ + Set us up as a Recipient + This will install a new caller_token into the FPS section. + This should really only be called to regenerate the recipient token. + """ + response = self.install_payment_instruction("MyRole=='Recipient';", token_type=token_type, transaction_id=transaction_id) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + recipient_token = rs.TokenId + try: + boto.config.save_system_option("FPS", "recipient_token", recipient_token) + except(IOError): + boto.config.save_user_option("FPS", "recipient_token", recipient_token) + + return recipient_token + else: + raise FPSResponseError(response.status, response.reason, body) + + def make_url(self, returnURL, paymentReason, pipelineName, transactionAmount, **params): + """ + Generate the URL with the signature required for a transaction + """ + # use the sandbox authorization endpoint if we're using the + # sandbox for API calls. + endpoint_host = 'authorize.payments.amazon.com' + if 'sandbox' in self.host: + endpoint_host = 'authorize.payments-sandbox.amazon.com' + base = "/cobranded-ui/actions/start" + + + params['callerKey'] = str(self.aws_access_key_id) + params['returnURL'] = str(returnURL) + params['paymentReason'] = str(paymentReason) + params['pipelineName'] = pipelineName + params["signatureMethod"] = 'HmacSHA256' + params["signatureVersion"] = '2' + params["transactionAmount"] = transactionAmount + + if(not params.has_key('callerReference')): + params['callerReference'] = str(uuid.uuid4()) + + parts = '' + for k in sorted(params.keys()): + parts += "&%s=%s" % (k, urllib.quote(params[k], '~')) + + canonical = '\n'.join(['GET', + str(endpoint_host).lower(), + base, + parts[1:]]) + + signature = self._auth_handler.sign_string(canonical) + params["signature"] = signature + + urlsuffix = '' + for k in sorted(params.keys()): + urlsuffix += "&%s=%s" % (k, urllib.quote(params[k], '~')) + urlsuffix = urlsuffix[1:] # strip the first & + + fmt = "https://%(endpoint_host)s%(base)s?%(urlsuffix)s" + final = fmt % vars() + return final + + def pay(self, transactionAmount, senderTokenId, + recipientTokenId=None, callerTokenId=None, + chargeFeeTo="Recipient", + callerReference=None, senderReference=None, recipientReference=None, + senderDescription=None, recipientDescription=None, callerDescription=None, + metadata=None, transactionDate=None, reserve=False): + """ + Make a payment transaction. You must specify the amount. + This can also perform a Reserve request if 'reserve' is set to True. + """ + params = {} + params['SenderTokenId'] = senderTokenId + # this is for 2008-09-17 specification + params['TransactionAmount.Amount'] = str(transactionAmount) + params['TransactionAmount.CurrencyCode'] = "USD" + #params['TransactionAmount'] = str(transactionAmount) + params['ChargeFeeTo'] = chargeFeeTo + + params['RecipientTokenId'] = ( + recipientTokenId if recipientTokenId is not None + else boto.config.get("FPS", "recipient_token") + ) + params['CallerTokenId'] = ( + callerTokenId if callerTokenId is not None + else boto.config.get("FPS", "caller_token") + ) + if(transactionDate != None): + params['TransactionDate'] = transactionDate + if(senderReference != None): + params['SenderReference'] = senderReference + if(recipientReference != None): + params['RecipientReference'] = recipientReference + if(senderDescription != None): + params['SenderDescription'] = senderDescription + if(recipientDescription != None): + params['RecipientDescription'] = recipientDescription + if(callerDescription != None): + params['CallerDescription'] = callerDescription + if(metadata != None): + params['MetaData'] = metadata + if(callerReference == None): + callerReference = uuid.uuid4() + params['CallerReference'] = callerReference + + if reserve: + response = self.make_request("Reserve", params) + else: + response = self.make_request("Pay", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def get_transaction_status(self, transactionId): + """ + Returns the status of a given transaction. + """ + params = {} + params['TransactionId'] = transactionId + + response = self.make_request("GetTransactionStatus", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def cancel(self, transactionId, description=None): + """ + Cancels a reserved or pending transaction. + """ + params = {} + params['transactionId'] = transactionId + if(description != None): + params['description'] = description + + response = self.make_request("Cancel", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def settle(self, reserveTransactionId, transactionAmount=None): + """ + Charges for a reserved payment. + """ + params = {} + params['ReserveTransactionId'] = reserveTransactionId + if(transactionAmount != None): + params['TransactionAmount'] = transactionAmount + + response = self.make_request("Settle", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def refund(self, callerReference, transactionId, refundAmount=None, callerDescription=None): + """ + Refund a transaction. This refunds the full amount by default unless 'refundAmount' is specified. + """ + params = {} + params['CallerReference'] = callerReference + params['TransactionId'] = transactionId + if(refundAmount != None): + params['RefundAmount'] = refundAmount + if(callerDescription != None): + params['CallerDescription'] = callerDescription + + response = self.make_request("Refund", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def get_recipient_verification_status(self, recipientTokenId): + """ + Test that the intended recipient has a verified Amazon Payments account. + """ + params ={} + params['RecipientTokenId'] = recipientTokenId + + response = self.make_request("GetRecipientVerificationStatus", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def get_token_by_caller_reference(self, callerReference): + """ + Returns details about the token specified by 'callerReference'. + """ + params ={} + params['callerReference'] = callerReference + + response = self.make_request("GetTokenByCaller", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + def get_token_by_caller_token(self, tokenId): + """ + Returns details about the token specified by 'callerReference'. + """ + params ={} + params['TokenId'] = tokenId + + response = self.make_request("GetTokenByCaller", params) + body = response.read() + if(response.status == 200): + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise FPSResponseError(response.status, response.reason, body) + + def verify_signature(self, end_point_url, http_parameters): + params = dict( + UrlEndPoint = end_point_url, + HttpParameters = http_parameters, + ) + response = self.make_request("VerifySignature", params) + body = response.read() + if(response.status != 200): + raise FPSResponseError(response.status, response.reason, body) + rs = ResultSet() + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs diff --git a/backup/src/boto/fps/test/__init__.py b/backup/src/boto/fps/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/src/boto/fps/test/test_install_caller_instruction.py b/backup/src/boto/fps/test/test_install_caller_instruction.py new file mode 100644 index 0000000..45df867 --- /dev/null +++ b/backup/src/boto/fps/test/test_install_caller_instruction.py @@ -0,0 +1,4 @@ +from boto.fps.connection import FPSConnection +conn = FPSConnection() +conn.install_caller_instruction() +conn.install_recipient_instruction() diff --git a/backup/src/boto/fps/test/test_verify_signature.py b/backup/src/boto/fps/test/test_verify_signature.py new file mode 100644 index 0000000..d59b4fb --- /dev/null +++ b/backup/src/boto/fps/test/test_verify_signature.py @@ -0,0 +1,6 @@ +from boto.fps.connection import FPSConnection +conn = FPSConnection() +# example response from the docs +params = 'expiry=08%2F2015&signature=ynDukZ9%2FG77uSJVb5YM0cadwHVwYKPMKOO3PNvgADbv6VtymgBxeOWEhED6KGHsGSvSJnMWDN%2FZl639AkRe9Ry%2F7zmn9CmiM%2FZkp1XtshERGTqi2YL10GwQpaH17MQqOX3u1cW4LlyFoLy4celUFBPq1WM2ZJnaNZRJIEY%2FvpeVnCVK8VIPdY3HMxPAkNi5zeF2BbqH%2BL2vAWef6vfHkNcJPlOuOl6jP4E%2B58F24ni%2B9ek%2FQH18O4kw%2FUJ7ZfKwjCCI13%2BcFybpofcKqddq8CuUJj5Ii7Pdw1fje7ktzHeeNhF0r9siWcYmd4JaxTP3NmLJdHFRq2T%2FgsF3vK9m3gw%3D%3D&signatureVersion=2&signatureMethod=RSA-SHA1&certificateUrl=https%3A%2F%2Ffps.sandbox.amazonaws.com%2Fcerts%2F090909%2FPKICert.pem&tokenID=A5BB3HUNAZFJ5CRXIPH72LIODZUNAUZIVP7UB74QNFQDSQ9MN4HPIKISQZWPLJXF&status=SC&callerReference=callerReferenceMultiUse1' +endpoint = 'http://vamsik.desktop.amazon.com:8080/ipn.jsp' +conn.verify_signature(endpoint, params) diff --git a/backup/src/boto/gs/__init__.py b/backup/src/boto/gs/__init__.py new file mode 100644 index 0000000..bf4c0b9 --- /dev/null +++ b/backup/src/boto/gs/__init__.py @@ -0,0 +1,22 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/backup/src/boto/gs/acl.py b/backup/src/boto/gs/acl.py new file mode 100644 index 0000000..33aaadf --- /dev/null +++ b/backup/src/boto/gs/acl.py @@ -0,0 +1,276 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.gs.user import User +from boto.exception import InvalidAclError + +ACCESS_CONTROL_LIST = 'AccessControlList' +ALL_AUTHENTICATED_USERS = 'AllAuthenticatedUsers' +ALL_USERS = 'AllUsers' +DOMAIN = 'Domain' +EMAIL_ADDRESS = 'EmailAddress' +ENTRY = 'Entry' +ENTRIES = 'Entries' +GROUP_BY_DOMAIN = 'GroupByDomain' +GROUP_BY_EMAIL = 'GroupByEmail' +GROUP_BY_ID = 'GroupById' +ID = 'ID' +NAME = 'Name' +OWNER = 'Owner' +PERMISSION = 'Permission' +SCOPE = 'Scope' +TYPE = 'type' +USER_BY_EMAIL = 'UserByEmail' +USER_BY_ID = 'UserById' + + +CannedACLStrings = ['private', 'public-read', + 'public-read-write', 'authenticated-read', + 'bucket-owner-read', 'bucket-owner-full-control'] + +SupportedPermissions = ['READ', 'WRITE', 'FULL_CONTROL'] + +class ACL: + + def __init__(self, parent=None): + self.parent = parent + self.entries = [] + + def __repr__(self): + # Owner is optional in GS ACLs. + if hasattr(self, 'owner'): + entries_repr = [''] + else: + entries_repr = ['Owner:%s' % self.owner.__repr__()] + acl_entries = self.entries + if acl_entries: + for e in acl_entries.entry_list: + entries_repr.append(e.__repr__()) + return '<%s>' % ', '.join(entries_repr) + + # Method with same signature as boto.s3.acl.ACL.add_email_grant(), to allow + # polymorphic treatment at application layer. + def add_email_grant(self, permission, email_address): + entry = Entry(type=USER_BY_EMAIL, email_address=email_address, + permission=permission) + self.entries.entry_list.append(entry) + + # Method with same signature as boto.s3.acl.ACL.add_user_grant(), to allow + # polymorphic treatment at application layer. + def add_user_grant(self, permission, user_id): + entry = Entry(permission=permission, type=USER_BY_ID, id=user_id) + self.entries.entry_list.append(entry) + + def add_group_email_grant(self, permission, email_address): + entry = Entry(type=GROUP_BY_EMAIL, email_address=email_address, + permission=permission) + self.entries.entry_list.append(entry) + + def add_group_grant(self, permission, group_id): + entry = Entry(type=GROUP_BY_ID, id=group_id, permission=permission) + self.entries.entry_list.append(entry) + + def startElement(self, name, attrs, connection): + if name == OWNER: + self.owner = User(self) + return self.owner + elif name == ENTRIES: + self.entries = Entries(self) + return self.entries + else: + return None + + def endElement(self, name, value, connection): + if name == OWNER: + pass + elif name == ENTRIES: + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '<%s>' % ACCESS_CONTROL_LIST + # Owner is optional in GS ACLs. + if hasattr(self, 'owner'): + s += self.owner.to_xml() + acl_entries = self.entries + if acl_entries: + s += acl_entries.to_xml() + s += '' % ACCESS_CONTROL_LIST + return s + + +class Entries: + + def __init__(self, parent=None): + self.parent = parent + # Entries is the class that represents the same-named XML + # element. entry_list is the list within this class that holds the data. + self.entry_list = [] + + def __repr__(self): + entries_repr = [] + for e in self.entry_list: + entries_repr.append(e.__repr__()) + return '' % ', '.join(entries_repr) + + def startElement(self, name, attrs, connection): + if name == ENTRY: + entry = Entry(self) + self.entry_list.append(entry) + return entry + else: + return None + + def endElement(self, name, value, connection): + if name == ENTRY: + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '<%s>' % ENTRIES + for entry in self.entry_list: + s += entry.to_xml() + s += '' % ENTRIES + return s + + +# Class that represents a single (Scope, Permission) entry in an ACL. +class Entry: + + def __init__(self, scope=None, type=None, id=None, name=None, + email_address=None, domain=None, permission=None): + if not scope: + scope = Scope(self, type, id, name, email_address, domain) + self.scope = scope + self.permission = permission + + def __repr__(self): + return '<%s: %s>' % (self.scope.__repr__(), self.permission.__repr__()) + + def startElement(self, name, attrs, connection): + if name == SCOPE: + if not TYPE in attrs: + raise InvalidAclError('Missing "%s" in "%s" part of ACL' % + (TYPE, SCOPE)) + self.scope = Scope(self, attrs[TYPE]) + return self.scope + elif name == PERMISSION: + pass + else: + return None + + def endElement(self, name, value, connection): + if name == SCOPE: + pass + elif name == PERMISSION: + value = value.strip() + if not value in SupportedPermissions: + raise InvalidAclError('Invalid Permission "%s"' % value) + self.permission = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '<%s>' % ENTRY + s += self.scope.to_xml() + s += '<%s>%s' % (PERMISSION, self.permission, PERMISSION) + s += '' % ENTRY + return s + +class Scope: + + # Map from Scope type to list of allowed sub-elems. + ALLOWED_SCOPE_TYPE_SUB_ELEMS = { + ALL_AUTHENTICATED_USERS : [], + ALL_USERS : [], + GROUP_BY_DOMAIN : [DOMAIN], + GROUP_BY_EMAIL : [EMAIL_ADDRESS, NAME], + GROUP_BY_ID : [ID, NAME], + USER_BY_EMAIL : [EMAIL_ADDRESS, NAME], + USER_BY_ID : [ID, NAME] + } + + def __init__(self, parent, type=None, id=None, name=None, + email_address=None, domain=None): + self.parent = parent + self.type = type + self.name = name + self.id = id + self.domain = domain + self.email_address = email_address + if not self.ALLOWED_SCOPE_TYPE_SUB_ELEMS.has_key(self.type): + raise InvalidAclError('Invalid %s %s "%s" ' % + (SCOPE, TYPE, self.type)) + + def __repr__(self): + named_entity = None + if self.id: + named_entity = self.id + elif self.email_address: + named_entity = self.email_address + elif self.domain: + named_entity = self.domain + if named_entity: + return '<%s: %s>' % (self.type, named_entity) + else: + return '<%s>' % self.type + + def startElement(self, name, attrs, connection): + if not name in self.ALLOWED_SCOPE_TYPE_SUB_ELEMS[self.type]: + raise InvalidAclError('Element "%s" not allowed in %s %s "%s" ' % + (name, SCOPE, TYPE, self.type)) + return None + + def endElement(self, name, value, connection): + value = value.strip() + if name == DOMAIN: + self.domain = value + elif name == EMAIL_ADDRESS: + self.email_address = value + elif name == ID: + self.id = value + elif name == NAME: + self.name = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '<%s type="%s">' % (SCOPE, self.type) + if self.type == ALL_AUTHENTICATED_USERS or self.type == ALL_USERS: + pass + elif self.type == GROUP_BY_DOMAIN: + s += '<%s>%s' % (DOMAIN, self.domain, DOMAIN) + elif self.type == GROUP_BY_EMAIL or self.type == USER_BY_EMAIL: + s += '<%s>%s' % (EMAIL_ADDRESS, self.email_address, + EMAIL_ADDRESS) + if self.name: + s += '<%s>%s' % (NAME, self.name, NAME) + elif self.type == GROUP_BY_ID or self.type == USER_BY_ID: + s += '<%s>%s' % (ID, self.id, ID) + if self.name: + s += '<%s>%s' % (NAME, self.name, NAME) + else: + raise InvalidAclError('Invalid scope type "%s" ', self.type) + + s += '' % SCOPE + return s diff --git a/backup/src/boto/gs/bucket.py b/backup/src/boto/gs/bucket.py new file mode 100644 index 0000000..b4b80e8 --- /dev/null +++ b/backup/src/boto/gs/bucket.py @@ -0,0 +1,173 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto import handler +from boto.exception import InvalidAclError +from boto.gs.acl import ACL +from boto.gs.acl import SupportedPermissions as GSPermissions +from boto.gs.key import Key as GSKey +from boto.s3.acl import Policy +from boto.s3.bucket import Bucket as S3Bucket +import xml.sax + +class Bucket(S3Bucket): + + def __init__(self, connection=None, name=None, key_class=GSKey): + super(Bucket, self).__init__(connection, name, key_class) + + def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): + if isinstance(acl_or_str, Policy): + raise InvalidAclError('Attempt to set S3 Policy on GS ACL') + elif isinstance(acl_or_str, ACL): + self.set_xml_acl(acl_or_str.to_xml(), key_name, headers=headers) + else: + self.set_canned_acl(acl_or_str, key_name, headers=headers) + + def get_acl(self, key_name='', headers=None, version_id=None): + response = self.connection.make_request('GET', self.name, key_name, + query_args='acl', headers=headers) + body = response.read() + if response.status == 200: + acl = ACL(self) + h = handler.XmlHandler(acl, self) + xml.sax.parseString(body, h) + return acl + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + # Method with same signature as boto.s3.bucket.Bucket.add_email_grant(), + # to allow polymorphic treatment at application layer. + def add_email_grant(self, permission, email_address, + recursive=False, headers=None): + """ + Convenience method that provides a quick way to add an email grant + to a bucket. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL + and then PUT's the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ, WRITE, FULL_CONTROL). + + :type email_address: string + :param email_address: The email address associated with the GS + account your are granting the permission to. + + :type recursive: boolean + :param recursive: A boolean value to controls whether the call + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + if permission not in GSPermissions: + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) + acl = self.get_acl(headers=headers) + acl.add_email_grant(permission, email_address) + self.set_acl(acl, headers=headers) + if recursive: + for key in self: + key.add_email_grant(permission, email_address, headers=headers) + + # Method with same signature as boto.s3.bucket.Bucket.add_user_grant(), + # to allow polymorphic treatment at application layer. + def add_user_grant(self, permission, user_id, recursive=False, headers=None): + """ + Convenience method that provides a quick way to add a canonical user grant to a bucket. + This method retrieves the current ACL, creates a new grant based on the parameters + passed in, adds that grant to the ACL and then PUTs the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ|WRITE|FULL_CONTROL) + + :type user_id: string + :param user_id: The canonical user id associated with the GS account you are granting + the permission to. + + :type recursive: bool + :param recursive: A boolean value to controls whether the call + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + if permission not in GSPermissions: + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) + acl = self.get_acl(headers=headers) + acl.add_user_grant(permission, user_id) + self.set_acl(acl, headers=headers) + if recursive: + for key in self: + key.add_user_grant(permission, user_id, headers=headers) + + def add_group_email_grant(self, permission, email_address, recursive=False, + headers=None): + """ + Convenience method that provides a quick way to add an email group + grant to a bucket. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL and + then PUT's the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|WRITE|FULL_CONTROL + See http://code.google.com/apis/storage/docs/developer-guide.html#authorization + for more details on permissions. + + :type email_address: string + :param email_address: The email address associated with the Google + Group to which you are granting the permission. + + :type recursive: bool + :param recursive: A boolean value to controls whether the call + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + if permission not in GSPermissions: + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) + acl = self.get_acl(headers=headers) + acl.add_group_email_grant(permission, email_address) + self.set_acl(acl, headers=headers) + if recursive: + for key in self: + key.add_group_email_grant(permission, email_address, + headers=headers) + + # Method with same input signature as boto.s3.bucket.Bucket.list_grants() + # (but returning different object type), to allow polymorphic treatment + # at application layer. + def list_grants(self, headers=None): + acl = self.get_acl(headers=headers) + return acl.entries diff --git a/backup/src/boto/gs/connection.py b/backup/src/boto/gs/connection.py new file mode 100644 index 0000000..ec81f32 --- /dev/null +++ b/backup/src/boto/gs/connection.py @@ -0,0 +1,39 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.connection import S3Connection +from boto.s3.connection import SubdomainCallingFormat +from boto.gs.bucket import Bucket + +class GSConnection(S3Connection): + + DefaultHost = 'commondatastorage.googleapis.com' + QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' + + def __init__(self, gs_access_key_id=None, gs_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host=DefaultHost, debug=0, https_connection_factory=None, + calling_format=SubdomainCallingFormat(), path='/'): + S3Connection.__init__(self, gs_access_key_id, gs_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, calling_format, path, + "google", Bucket) diff --git a/backup/src/boto/gs/key.py b/backup/src/boto/gs/key.py new file mode 100644 index 0000000..608a9a5 --- /dev/null +++ b/backup/src/boto/gs/key.py @@ -0,0 +1,247 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.key import Key as S3Key + +class Key(S3Key): + + def add_email_grant(self, permission, email_address): + """ + Convenience method that provides a quick way to add an email grant to a + key. This method retrieves the current ACL, creates a new grant based on + the parameters passed in, adds that grant to the ACL and then PUT's the + new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|FULL_CONTROL + See http://code.google.com/apis/storage/docs/developer-guide.html#authorization + for more details on permissions. + + :type email_address: string + :param email_address: The email address associated with the Google + account to which you are granting the permission. + """ + acl = self.get_acl() + acl.add_email_grant(permission, email_address) + self.set_acl(acl) + + def add_user_grant(self, permission, user_id): + """ + Convenience method that provides a quick way to add a canonical user + grant to a key. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL and + then PUT's the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|FULL_CONTROL + See http://code.google.com/apis/storage/docs/developer-guide.html#authorization + for more details on permissions. + + :type user_id: string + :param user_id: The canonical user id associated with the GS account to + which you are granting the permission. + """ + acl = self.get_acl() + acl.add_user_grant(permission, user_id) + self.set_acl(acl) + + def add_group_email_grant(self, permission, email_address, headers=None): + """ + Convenience method that provides a quick way to add an email group + grant to a key. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL and + then PUT's the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|FULL_CONTROL + See http://code.google.com/apis/storage/docs/developer-guide.html#authorization + for more details on permissions. + + :type email_address: string + :param email_address: The email address associated with the Google + Group to which you are granting the permission. + """ + acl = self.get_acl(headers=headers) + acl.add_group_email_grant(permission, email_address) + self.set_acl(acl, headers=headers) + + def add_group_grant(self, permission, group_id): + """ + Convenience method that provides a quick way to add a canonical group + grant to a key. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL and + then PUT's the new ACL back to GS. + + :type permission: string + :param permission: The permission being granted. Should be one of: + READ|FULL_CONTROL + See http://code.google.com/apis/storage/docs/developer-guide.html#authorization + for more details on permissions. + + :type group_id: string + :param group_id: The canonical group id associated with the Google + Groups account you are granting the permission to. + """ + acl = self.get_acl() + acl.add_group_grant(permission, group_id) + self.set_acl(acl) + + def set_contents_from_file(self, fp, headers={}, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + res_upload_handler=None): + """ + Store an object in GS using the name of the Key object as the + key in GS and the contents of the file pointed to by 'fp' as the + contents. + + :type fp: file + :param fp: the file whose contents are to be uploaded + + :type headers: dict + :param headers: additional HTTP headers to be sent with the PUT request. + + :type replace: bool + :param replace: If this parameter is False, the method will first check + to see if an object exists in the bucket with the same key. If it + does, it won't overwrite it. The default value is True which will + overwrite the object. + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to GS and the second representing the + total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the cb + parameter, this parameter determines the granularity of the callback + by defining the maximum number of times the callback will be called + during the file transfer. + + :type policy: :class:`boto.gs.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key + in GS. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum + of the file as the first element and the Base64-encoded version of + the plain checksum as the second element. This is the same format + returned by the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to + upload, it's silly to have to do it twice so this param, if present, + will be used as the MD5 values of the file. Otherwise, the checksum + will be computed. + + :type res_upload_handler: ResumableUploadHandler + :param res_upload_handler: If provided, this handler will perform the + upload. + + TODO: At some point we should refactor the Bucket and Key classes, + to move functionality common to all providers into a parent class, + and provider-specific functionality into subclasses (rather than + just overriding/sharing code the way it currently works). + """ + provider = self.bucket.connection.provider + if headers is None: + headers = {} + if policy: + headers[provider.acl_header] = policy + if hasattr(fp, 'name'): + self.path = fp.name + if self.bucket != None: + if not md5: + md5 = self.compute_md5(fp) + else: + # Even if md5 is provided, still need to set size of content. + fp.seek(0, 2) + self.size = fp.tell() + fp.seek(0) + self.md5 = md5[0] + self.base64md5 = md5[1] + if self.name == None: + self.name = self.md5 + if not replace: + k = self.bucket.lookup(self.name) + if k: + return + if res_upload_handler: + res_upload_handler.send_file(self, fp, headers, cb, num_cb) + else: + # Not a resumable transfer so use basic send_file mechanism. + self.send_file(fp, headers, cb, num_cb) + + def set_contents_from_filename(self, filename, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + reduced_redundancy=None, + res_upload_handler=None): + """ + Store an object in GS using the name of the Key object as the + key in GS and the contents of the file named by 'filename'. + See set_contents_from_file method for details about the + parameters. + + :type filename: string + :param filename: The name of the file that you want to put onto GS + + :type headers: dict + :param headers: Additional headers to pass along with the request to GS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it + already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from GS and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb + parameter this parameter determines the granularity of the callback + by defining the maximum number of times the callback will be called + during the file transfer. + + :type policy: :class:`boto.gs.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key + in GS. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum + of the file as the first element and the Base64-encoded version of + the plain checksum as the second element. This is the same format + returned by the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to + upload, it's silly to have to do it twice so this param, if present, + will be used as the MD5 values of the file. Otherwise, the checksum + will be computed. + + :type res_upload_handler: ResumableUploadHandler + :param res_upload_handler: If provided, this handler will perform the + upload. + """ + fp = open(filename, 'rb') + self.set_contents_from_file(fp, headers, replace, cb, num_cb, + policy, md5, res_upload_handler) + fp.close() diff --git a/backup/src/boto/gs/resumable_upload_handler.py b/backup/src/boto/gs/resumable_upload_handler.py new file mode 100644 index 0000000..e8d5b03 --- /dev/null +++ b/backup/src/boto/gs/resumable_upload_handler.py @@ -0,0 +1,526 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import cgi +import errno +import httplib +import os +import re +import socket +import time +import urlparse +import boto +from boto import config +from boto.connection import AWSAuthConnection +from boto.exception import InvalidUriError +from boto.exception import ResumableTransferDisposition +from boto.exception import ResumableUploadException + +""" +Handler for Google Storage resumable uploads. See +http://code.google.com/apis/storage/docs/developer-guide.html#resumable +for details. + +Resumable uploads will retry failed uploads, resuming at the byte +count completed by the last upload attempt. If too many retries happen with +no progress (per configurable num_retries param), the upload will be aborted. + +The caller can optionally specify a tracker_file_name param in the +ResumableUploadHandler constructor. If you do this, that file will +save the state needed to allow retrying later, in a separate process +(e.g., in a later run of gsutil). +""" + + +class ResumableUploadHandler(object): + + BUFFER_SIZE = 8192 + RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error, + socket.gaierror) + + # (start, end) response indicating server has nothing (upload protocol uses + # inclusive numbering). + SERVER_HAS_NOTHING = (0, -1) + + def __init__(self, tracker_file_name=None, num_retries=None): + """ + Constructor. Instantiate once for each uploaded file. + + :type tracker_file_name: string + :param tracker_file_name: optional file name to save tracker URI. + If supplied and the current process fails the upload, it can be + retried in a new process. If called with an existing file containing + a valid tracker URI, we'll resume the upload from this URI; else + we'll start a new resumable upload (and write the URI to this + tracker file). + + :type num_retries: int + :param num_retries: the number of times we'll re-try a resumable upload + making no progress. (Count resets every time we get progress, so + upload can span many more than this number of retries.) + """ + self.tracker_file_name = tracker_file_name + self.num_retries = num_retries + self.server_has_bytes = 0 # Byte count at last server check. + self.tracker_uri = None + if tracker_file_name: + self._load_tracker_uri_from_file() + # Save upload_start_point in instance state so caller can find how + # much was transferred by this ResumableUploadHandler (across retries). + self.upload_start_point = None + + def _load_tracker_uri_from_file(self): + f = None + try: + f = open(self.tracker_file_name, 'r') + uri = f.readline().strip() + self._set_tracker_uri(uri) + except IOError, e: + # Ignore non-existent file (happens first time an upload + # is attempted on a file), but warn user for other errors. + if e.errno != errno.ENOENT: + # Will restart because self.tracker_uri == None. + print('Couldn\'t read URI tracker file (%s): %s. Restarting ' + 'upload from scratch.' % + (self.tracker_file_name, e.strerror)) + except InvalidUriError, e: + # Warn user, but proceed (will restart because + # self.tracker_uri == None). + print('Invalid tracker URI (%s) found in URI tracker file ' + '(%s). Restarting upload from scratch.' % + (uri, self.tracker_file_name)) + finally: + if f: + f.close() + + def _save_tracker_uri_to_file(self): + """ + Saves URI to tracker file if one was passed to constructor. + """ + if not self.tracker_file_name: + return + f = None + try: + f = open(self.tracker_file_name, 'w') + f.write(self.tracker_uri) + except IOError, e: + raise ResumableUploadException( + 'Couldn\'t write URI tracker file (%s): %s.\nThis can happen' + 'if you\'re using an incorrectly configured upload tool\n' + '(e.g., gsutil configured to save tracker files to an ' + 'unwritable directory)' % + (self.tracker_file_name, e.strerror), + ResumableTransferDisposition.ABORT) + finally: + if f: + f.close() + + def _set_tracker_uri(self, uri): + """ + Called when we start a new resumable upload or get a new tracker + URI for the upload. Saves URI and resets upload state. + + Raises InvalidUriError if URI is syntactically invalid. + """ + parse_result = urlparse.urlparse(uri) + if (parse_result.scheme.lower() not in ['http', 'https'] or + not parse_result.netloc or not parse_result.query): + raise InvalidUriError('Invalid tracker URI (%s)' % uri) + qdict = cgi.parse_qs(parse_result.query) + if not qdict or not 'upload_id' in qdict: + raise InvalidUriError('Invalid tracker URI (%s)' % uri) + self.tracker_uri = uri + self.tracker_uri_host = parse_result.netloc + self.tracker_uri_path = '%s/?%s' % (parse_result.netloc, + parse_result.query) + self.server_has_bytes = 0 + + def get_tracker_uri(self): + """ + Returns upload tracker URI, or None if the upload has not yet started. + """ + return self.tracker_uri + + def _remove_tracker_file(self): + if (self.tracker_file_name and + os.path.exists(self.tracker_file_name)): + os.unlink(self.tracker_file_name) + + def _build_content_range_header(self, range_spec='*', length_spec='*'): + return 'bytes %s/%s' % (range_spec, length_spec) + + def _query_server_state(self, conn, file_length): + """ + Queries server to find out what bytes it currently has. + + Note that this method really just makes special case use of the + fact that the upload server always returns the current start/end + state whenever a PUT doesn't complete. + + Returns (server_start, server_end), where the values are inclusive. + For example, (0, 2) would mean that the server has bytes 0, 1, *and* 2. + + Raises ResumableUploadException if problem querying server. + """ + # Send an empty PUT so that server replies with this resumable + # transfer's state. + put_headers = {} + put_headers['Content-Range'] = ( + self._build_content_range_header('*', file_length)) + put_headers['Content-Length'] = '0' + resp = AWSAuthConnection.make_request(conn, 'PUT', + path=self.tracker_uri_path, + auth_path=self.tracker_uri_path, + headers=put_headers, + host=self.tracker_uri_host) + if resp.status == 200: + return (0, file_length) # Completed upload. + if resp.status != 308: + # This means the server didn't have any state for the given + # upload ID, which can happen (for example) if the caller saved + # the tracker URI to a file and then tried to restart the transfer + # after that upload ID has gone stale. In that case we need to + # start a new transfer (and the caller will then save the new + # tracker URI to the tracker file). + raise ResumableUploadException( + 'Got non-308 response (%s) from server state query' % + resp.status, ResumableTransferDisposition.START_OVER) + got_valid_response = False + range_spec = resp.getheader('range') + if range_spec: + # Parse 'bytes=-' range_spec. + m = re.search('bytes=(\d+)-(\d+)', range_spec) + if m: + server_start = long(m.group(1)) + server_end = long(m.group(2)) + got_valid_response = True + else: + # No Range header, which means the server does not yet have + # any bytes. Note that the Range header uses inclusive 'from' + # and 'to' values. Since Range 0-0 would mean that the server + # has byte 0, omitting the Range header is used to indicate that + # the server doesn't have any bytes. + return self.SERVER_HAS_NOTHING + if not got_valid_response: + raise ResumableUploadException( + 'Couldn\'t parse upload server state query response (%s)' % + str(resp.getheaders()), ResumableTransferDisposition.START_OVER) + if conn.debug >= 1: + print 'Server has: Range: %d - %d.' % (server_start, server_end) + return (server_start, server_end) + + def _start_new_resumable_upload(self, key, headers=None): + """ + Starts a new resumable upload. + + Raises ResumableUploadException if any errors occur. + """ + conn = key.bucket.connection + if conn.debug >= 1: + print 'Starting new resumable upload.' + self.server_has_bytes = 0 + + # Start a new resumable upload by sending a POST request with an + # empty body and the "X-Goog-Resumable: start" header. Include any + # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length + # (and raise an exception if they tried to pass one, since it's + # a semantic error to specify it at this point, and if we were to + # include one now it would cause the server to expect that many + # bytes; the POST doesn't include the actual file bytes We set + # the Content-Length in the subsequent PUT, based on the uploaded + # file size. + post_headers = {} + for k in headers: + if k.lower() == 'content-length': + raise ResumableUploadException( + 'Attempt to specify Content-Length header (disallowed)', + ResumableTransferDisposition.ABORT) + post_headers[k] = headers[k] + post_headers[conn.provider.resumable_upload_header] = 'start' + + resp = conn.make_request( + 'POST', key.bucket.name, key.name, post_headers) + # Get tracker URI from response 'Location' header. + body = resp.read() + # Check for '201 Created' response code. + if resp.status != 201: + raise ResumableUploadException( + 'Got status %d from attempt to start resumable upload' % + resp.status, ResumableTransferDisposition.WAIT_BEFORE_RETRY) + tracker_uri = resp.getheader('Location') + if not tracker_uri: + raise ResumableUploadException( + 'No resumable tracker URI found in resumable initiation ' + 'POST response (%s)' % body, + ResumableTransferDisposition.WAIT_BEFORE_RETRY) + self._set_tracker_uri(tracker_uri) + self._save_tracker_uri_to_file() + + def _upload_file_bytes(self, conn, http_conn, fp, file_length, + total_bytes_uploaded, cb, num_cb): + """ + Makes one attempt to upload file bytes, using an existing resumable + upload connection. + + Returns etag from server upon success. + + Raises ResumableUploadException if any problems occur. + """ + buf = fp.read(self.BUFFER_SIZE) + if cb: + if num_cb > 2: + cb_count = file_length / self.BUFFER_SIZE / (num_cb-2) + elif num_cb < 0: + cb_count = -1 + else: + cb_count = 0 + i = 0 + cb(total_bytes_uploaded, file_length) + + # Build resumable upload headers for the transfer. Don't send a + # Content-Range header if the file is 0 bytes long, because the + # resumable upload protocol uses an *inclusive* end-range (so, sending + # 'bytes 0-0/1' would actually mean you're sending a 1-byte file). + put_headers = {} + if file_length: + range_header = self._build_content_range_header( + '%d-%d' % (total_bytes_uploaded, file_length - 1), + file_length) + put_headers['Content-Range'] = range_header + # Set Content-Length to the total bytes we'll send with this PUT. + put_headers['Content-Length'] = str(file_length - total_bytes_uploaded) + http_request = AWSAuthConnection.build_base_http_request( + conn, 'PUT', path=self.tracker_uri_path, auth_path=None, + headers=put_headers, host=self.tracker_uri_host) + http_conn.putrequest('PUT', http_request.path) + for k in put_headers: + http_conn.putheader(k, put_headers[k]) + http_conn.endheaders() + + # Turn off debug on http connection so upload content isn't included + # in debug stream. + http_conn.set_debuglevel(0) + while buf: + http_conn.send(buf) + total_bytes_uploaded += len(buf) + if cb: + i += 1 + if i == cb_count or cb_count == -1: + cb(total_bytes_uploaded, file_length) + i = 0 + buf = fp.read(self.BUFFER_SIZE) + if cb: + cb(total_bytes_uploaded, file_length) + if total_bytes_uploaded != file_length: + raise ResumableUploadException('File changed during upload: EOF at ' + '%d bytes of %d byte file.' % + (total_bytes_uploaded, file_length), + ResumableTransferDisposition.ABORT) + resp = http_conn.getresponse() + body = resp.read() + # Restore http connection debug level. + http_conn.set_debuglevel(conn.debug) + + additional_note = '' + if resp.status == 200: + return resp.getheader('etag') # Success + # Retry status 503 errors after a delay. + elif resp.status == 503: + disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY + elif resp.status == 500: + disposition = ResumableTransferDisposition.ABORT + additional_note = ('This can happen if you attempt to upload a ' + 'different size file on a already partially ' + 'uploaded resumable upload') + else: + disposition = ResumableTransferDisposition.ABORT + raise ResumableUploadException('Got response code %d while attempting ' + 'upload (%s)%s' % + (resp.status, resp.reason, + additional_note), disposition) + + def _attempt_resumable_upload(self, key, fp, file_length, headers, cb, + num_cb): + """ + Attempts a resumable upload. + + Returns etag from server upon success. + + Raises ResumableUploadException if any problems occur. + """ + (server_start, server_end) = self.SERVER_HAS_NOTHING + conn = key.bucket.connection + if self.tracker_uri: + # Try to resume existing resumable upload. + try: + (server_start, server_end) = ( + self._query_server_state(conn, file_length)) + self.server_has_bytes = server_start + if conn.debug >= 1: + print 'Resuming transfer.' + except ResumableUploadException, e: + if conn.debug >= 1: + print 'Unable to resume transfer (%s).' % e.message + self._start_new_resumable_upload(key, headers) + else: + self._start_new_resumable_upload(key, headers) + + # upload_start_point allows the code that instantiated the + # ResumableUploadHandler to find out the point from which it started + # uploading (e.g., so it can correctly compute throughput). + if self.upload_start_point is None: + self.upload_start_point = server_end + + if server_end == file_length: + return # Done. + total_bytes_uploaded = server_end + 1 + fp.seek(total_bytes_uploaded) + conn = key.bucket.connection + + # Get a new HTTP connection (vs conn.get_http_connection(), which reuses + # pool connections) because httplib requires a new HTTP connection per + # transaction. (Without this, calling http_conn.getresponse() would get + # "ResponseNotReady".) + http_conn = conn.new_http_connection(self.tracker_uri_host, + conn.is_secure) + http_conn.set_debuglevel(conn.debug) + + # Make sure to close http_conn at end so if a local file read + # failure occurs partway through server will terminate current upload + # and can report that progress on next attempt. + try: + return self._upload_file_bytes(conn, http_conn, fp, file_length, + total_bytes_uploaded, cb, num_cb) + finally: + http_conn.close() + + def _check_final_md5(self, key, etag): + """ + Checks that etag from server agrees with md5 computed before upload. + This is important, since the upload could have spanned a number of + hours and multiple processes (e.g., gsutil runs), and the user could + change some of the file and not realize they have inconsistent data. + """ + if key.bucket.connection.debug >= 1: + print 'Checking md5 against etag.' + if key.md5 != etag.strip('"\''): + # Call key.open_read() before attempting to delete the + # (incorrect-content) key, so we perform that request on a + # different HTTP connection. This is neededb because httplib + # will return a "Response not ready" error if you try to perform + # a second transaction on the connection. + key.open_read() + key.close() + key.delete() + raise ResumableUploadException( + 'File changed during upload: md5 signature doesn\'t match etag ' + '(incorrect uploaded object deleted)', + ResumableTransferDisposition.ABORT) + + def send_file(self, key, fp, headers, cb=None, num_cb=10): + """ + Upload a file to a key into a bucket on GS, using GS resumable upload + protocol. + + :type key: :class:`boto.s3.key.Key` or subclass + :param key: The Key object to which data is to be uploaded + + :type fp: file-like object + :param fp: The file pointer to upload + + :type headers: dict + :param headers: The headers to pass along with the PUT request + + :type cb: function + :param cb: a callback function that will be called to report progress on + the upload. The callback should accept two integer parameters, the + first representing the number of bytes that have been successfully + transmitted to GS, and the second representing the total number of + bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the cb + parameter, this parameter determines the granularity of the callback + by defining the maximum number of times the callback will be called + during the file transfer. Providing a negative integer will cause + your callback to be called with each buffer read. + + Raises ResumableUploadException if a problem occurs during the transfer. + """ + + if not headers: + headers = {} + + fp.seek(0, os.SEEK_END) + file_length = fp.tell() + fp.seek(0) + debug = key.bucket.connection.debug + + # Use num-retries from constructor if one was provided; else check + # for a value specified in the boto config file; else default to 5. + if self.num_retries is None: + self.num_retries = config.getint('Boto', 'num_retries', 5) + progress_less_iterations = 0 + + while True: # Retry as long as we're making progress. + server_had_bytes_before_attempt = self.server_has_bytes + try: + etag = self._attempt_resumable_upload(key, fp, file_length, + headers, cb, num_cb) + # Upload succceded, so remove the tracker file (if have one). + self._remove_tracker_file() + self._check_final_md5(key, etag) + if debug >= 1: + print 'Resumable upload complete.' + return + except self.RETRYABLE_EXCEPTIONS, e: + if debug >= 1: + print('Caught exception (%s)' % e.__repr__()) + except ResumableUploadException, e: + if e.disposition == ResumableTransferDisposition.ABORT: + if debug >= 1: + print('Caught non-retryable ResumableUploadException ' + '(%s)' % e.message) + raise + else: + if debug >= 1: + print('Caught ResumableUploadException (%s) - will ' + 'retry' % e.message) + + # At this point we had a re-tryable failure; see if made progress. + if self.server_has_bytes > server_had_bytes_before_attempt: + progress_less_iterations = 0 + else: + progress_less_iterations += 1 + + if progress_less_iterations > self.num_retries: + # Don't retry any longer in the current process. + raise ResumableUploadException( + 'Too many resumable upload attempts failed without ' + 'progress. You might try this upload again later', + ResumableTransferDisposition.ABORT) + + sleep_time_secs = 2**progress_less_iterations + if debug >= 1: + print ('Got retryable failure (%d progress-less in a row).\n' + 'Sleeping %d seconds before re-trying' % + (progress_less_iterations, sleep_time_secs)) + time.sleep(sleep_time_secs) diff --git a/backup/src/boto/gs/user.py b/backup/src/boto/gs/user.py new file mode 100644 index 0000000..62f2cf5 --- /dev/null +++ b/backup/src/boto/gs/user.py @@ -0,0 +1,54 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class User: + def __init__(self, parent=None, id='', name=''): + if parent: + parent.owner = self + self.type = None + self.id = id + self.name = name + + def __repr__(self): + return self.id + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'ID': + self.id = value + else: + setattr(self, name, value) + + def to_xml(self, element_name='Owner'): + if self.type: + s = '<%s type="%s">' % (element_name, self.type) + else: + s = '<%s>' % element_name + s += '%s' % self.id + if self.name: + s += '%s' % self.name + s += '' % element_name + return s diff --git a/backup/src/boto/handler.py b/backup/src/boto/handler.py new file mode 100644 index 0000000..525f9c9 --- /dev/null +++ b/backup/src/boto/handler.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax + +class XmlHandler(xml.sax.ContentHandler): + + def __init__(self, root_node, connection): + self.connection = connection + self.nodes = [('root', root_node)] + self.current_text = '' + + def startElement(self, name, attrs): + self.current_text = '' + new_node = self.nodes[-1][1].startElement(name, attrs, self.connection) + if new_node != None: + self.nodes.append((name, new_node)) + + def endElement(self, name): + self.nodes[-1][1].endElement(name, self.current_text, self.connection) + if self.nodes[-1][0] == name: + self.nodes.pop() + self.current_text = '' + + def characters(self, content): + self.current_text += content + + diff --git a/backup/src/boto/iam/__init__.py b/backup/src/boto/iam/__init__.py new file mode 100644 index 0000000..498d736 --- /dev/null +++ b/backup/src/boto/iam/__init__.py @@ -0,0 +1,27 @@ +# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# this is here for backward compatibility +# originally, the IAMConnection class was defined here +from connection import IAMConnection + + diff --git a/backup/src/boto/iam/connection.py b/backup/src/boto/iam/connection.py new file mode 100644 index 0000000..39ab704 --- /dev/null +++ b/backup/src/boto/iam/connection.py @@ -0,0 +1,1006 @@ +# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +import boto.jsonresponse +from boto.connection import AWSQueryConnection + +#boto.set_stream_logger('iam') + +class IAMConnection(AWSQueryConnection): + + APIVersion = '2010-05-08' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, host='iam.amazonaws.com', + debug=0, https_connection_factory=None, path='/'): + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, + proxy_port, proxy_user, proxy_pass, + host, debug, https_connection_factory, path) + + def _required_auth_capability(self): + return ['iam'] + + def get_response(self, action, params, path='/', parent=None, + verb='GET', list_marker='Set'): + """ + Utility method to handle calls to IAM and parsing of responses. + """ + if not parent: + parent = self + response = self.make_request(action, params, path, verb) + body = response.read() + boto.log.debug(body) + if response.status == 200: + e = boto.jsonresponse.Element(list_marker=list_marker) + h = boto.jsonresponse.XmlHandler(e, parent) + h.parse(body) + return e + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + # + # Group methods + # + + def get_all_groups(self, path_prefix='/', marker=None, max_items=None): + """ + List the groups that have the specified path prefix. + + :type path_prefix: string + :param path_prefix: If provided, only groups whose paths match + the provided prefix will be returned. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {} + if path_prefix: + params['PathPrefix'] = path_prefix + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListGroups', params, + list_marker='Groups') + + def get_group(self, group_name, marker=None, max_items=None): + """ + Return a list of users that are in the specified group. + + :type group_name: string + :param group_name: The name of the group whose information should + be returned. + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'GroupName' : group_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('GetGroup', params, list_marker='Users') + + def create_group(self, group_name, path='/'): + """ + Create a group. + + :type group_name: string + :param group_name: The name of the new group + + :type path: string + :param path: The path to the group (Optional). Defaults to /. + + """ + params = {'GroupName' : group_name, + 'Path' : path} + return self.get_response('CreateGroup', params) + + def delete_group(self, group_name): + """ + Delete a group. The group must not contain any Users or + have any attached policies + + :type group_name: string + :param group_name: The name of the group to delete. + + """ + params = {'GroupName' : group_name} + return self.get_response('DeleteGroup', params) + + def update_group(self, group_name, new_group_name=None, new_path=None): + """ + Updates name and/or path of the specified group. + + :type group_name: string + :param group_name: The name of the new group + + :type new_group_name: string + :param new_group_name: If provided, the name of the group will be + changed to this name. + + :type new_path: string + :param new_path: If provided, the path of the group will be + changed to this path. + + """ + params = {'GroupName' : group_name} + if new_group_name: + params['NewGroupName'] = new_group_name + if new_path: + params['NewPath'] = new_path + return self.get_response('UpdateGroup', params) + + def add_user_to_group(self, group_name, user_name): + """ + Add a user to a group + + :type group_name: string + :param group_name: The name of the new group + + :type user_name: string + :param user_name: The to be added to the group. + + """ + params = {'GroupName' : group_name, + 'UserName' : user_name} + return self.get_response('AddUserToGroup', params) + + def remove_user_from_group(self, group_name, user_name): + """ + Remove a user from a group. + + :type group_name: string + :param group_name: The name of the new group + + :type user_name: string + :param user_name: The user to remove from the group. + + """ + params = {'GroupName' : group_name, + 'UserName' : user_name} + return self.get_response('RemoveUserFromGroup', params) + + def put_group_policy(self, group_name, policy_name, policy_json): + """ + Adds or updates the specified policy document for the specified group. + + :type group_name: string + :param group_name: The name of the group the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to get. + + :type policy_json: string + :param policy_json: The policy document. + + """ + params = {'GroupName' : group_name, + 'PolicyName' : policy_name, + 'PolicyDocument' : policy_json} + return self.get_response('PutGroupPolicy', params, verb='POST') + + def get_all_group_policies(self, group_name, marker=None, max_items=None): + """ + List the names of the policies associated with the specified group. + + :type group_name: string + :param group_name: The name of the group the policy is associated with. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'GroupName' : group_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListGroupPolicies', params, + list_marker='PolicyNames') + + def get_group_policy(self, group_name, policy_name): + """ + Retrieves the specified policy document for the specified group. + + :type group_name: string + :param group_name: The name of the group the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to get. + + """ + params = {'GroupName' : group_name, + 'PolicyName' : policy_name} + return self.get_response('GetGroupPolicy', params, verb='POST') + + def delete_group_policy(self, group_name, policy_name): + """ + Deletes the specified policy document for the specified group. + + :type group_name: string + :param group_name: The name of the group the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to delete. + + """ + params = {'GroupName' : group_name, + 'PolicyName' : policy_name} + return self.get_response('DeleteGroupPolicy', params, verb='POST') + + def get_all_users(self, path_prefix='/', marker=None, max_items=None): + """ + List the users that have the specified path prefix. + + :type path_prefix: string + :param path_prefix: If provided, only users whose paths match + the provided prefix will be returned. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'PathPrefix' : path_prefix} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListUsers', params, list_marker='Users') + + # + # User methods + # + + def create_user(self, user_name, path='/'): + """ + Create a user. + + :type user_name: string + :param user_name: The name of the new user + + :type path: string + :param path: The path in which the user will be created. + Defaults to /. + + """ + params = {'UserName' : user_name, + 'Path' : path} + return self.get_response('CreateUser', params) + + def delete_user(self, user_name): + """ + Delete a user including the user's path, GUID and ARN. + + If the user_name is not specified, the user_name is determined + implicitly based on the AWS Access Key ID used to sign the request. + + :type user_name: string + :param user_name: The name of the user to delete. + + """ + params = {'UserName' : user_name} + return self.get_response('DeleteUser', params) + + def get_user(self, user_name=None): + """ + Retrieve information about the specified user. + + If the user_name is not specified, the user_name is determined + implicitly based on the AWS Access Key ID used to sign the request. + + :type user_name: string + :param user_name: The name of the user to delete. + If not specified, defaults to user making + request. + + """ + params = {} + if user_name: + params['UserName'] = user_name + return self.get_response('GetUser', params) + + def update_user(self, user_name, new_user_name=None, new_path=None): + """ + Updates name and/or path of the specified user. + + :type user_name: string + :param user_name: The name of the user + + :type new_user_name: string + :param new_user_name: If provided, the username of the user will be + changed to this username. + + :type new_path: string + :param new_path: If provided, the path of the user will be + changed to this path. + + """ + params = {'UserName' : user_name} + if new_user_name: + params['NewUserName'] = new_user_name + if new_path: + params['NewPath'] = new_path + return self.get_response('UpdateUser', params) + + def get_all_user_policies(self, user_name, marker=None, max_items=None): + """ + List the names of the policies associated with the specified user. + + :type user_name: string + :param user_name: The name of the user the policy is associated with. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'UserName' : user_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListUserPolicies', params, + list_marker='PolicyNames') + + def put_user_policy(self, user_name, policy_name, policy_json): + """ + Adds or updates the specified policy document for the specified user. + + :type user_name: string + :param user_name: The name of the user the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to get. + + :type policy_json: string + :param policy_json: The policy document. + + """ + params = {'UserName' : user_name, + 'PolicyName' : policy_name, + 'PolicyDocument' : policy_json} + return self.get_response('PutUserPolicy', params, verb='POST') + + def get_user_policy(self, user_name, policy_name): + """ + Retrieves the specified policy document for the specified user. + + :type user_name: string + :param user_name: The name of the user the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to get. + + """ + params = {'UserName' : user_name, + 'PolicyName' : policy_name} + return self.get_response('GetUserPolicy', params, verb='POST') + + def delete_user_policy(self, user_name, policy_name): + """ + Deletes the specified policy document for the specified user. + + :type user_name: string + :param user_name: The name of the user the policy is associated with. + + :type policy_name: string + :param policy_name: The policy document to delete. + + """ + params = {'UserName' : user_name, + 'PolicyName' : policy_name} + return self.get_response('DeleteUserPolicy', params, verb='POST') + + def get_groups_for_user(self, user_name, marker=None, max_items=None): + """ + List the groups that a specified user belongs to. + + :type user_name: string + :param user_name: The name of the user to list groups for. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'UserName' : user_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListGroupsForUser', params, + list_marker='Groups') + + # + # Access Keys + # + + def get_all_access_keys(self, user_name, marker=None, max_items=None): + """ + Get all access keys associated with an account. + + :type user_name: string + :param user_name: The username of the new user + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + """ + params = {'UserName' : user_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListAccessKeys', params, + list_marker='AccessKeyMetadata') + + def create_access_key(self, user_name=None): + """ + Create a new AWS Secret Access Key and corresponding AWS Access Key ID + for the specified user. The default status for new keys is Active + + If the user_name is not specified, the user_name is determined + implicitly based on the AWS Access Key ID used to sign the request. + + :type user_name: string + :param user_name: The username of the new user + + """ + params = {'UserName' : user_name} + return self.get_response('CreateAccessKey', params) + + def update_access_key(self, access_key_id, status, user_name=None): + """ + Changes the status of the specified access key from Active to Inactive + or vice versa. This action can be used to disable a user's key as + part of a key rotation workflow. + + If the user_name is not specified, the user_name is determined + implicitly based on the AWS Access Key ID used to sign the request. + + :type access_key_id: string + :param access_key_id: The ID of the access key. + + :type status: string + :param status: Either Active or Inactive. + + :type user_name: string + :param user_name: The username of user (optional). + + """ + params = {'AccessKeyId' : access_key_id, + 'Status' : status} + if user_name: + params['UserName'] = user_name + return self.get_response('UpdateAccessKey', params) + + def delete_access_key(self, access_key_id, user_name=None): + """ + Delete an access key associated with a user. + + If the user_name is not specified, it is determined implicitly based + on the AWS Access Key ID used to sign the request. + + :type access_key_id: string + :param access_key_id: The ID of the access key to be deleted. + + :type user_name: string + :param user_name: The username of the new user + + """ + params = {'AccessKeyId' : access_key_id} + if user_name: + params['UserName'] = user_name + return self.get_response('DeleteAccessKey', params) + + # + # Signing Certificates + # + + def get_all_signing_certs(self, marker=None, max_items=None, + user_name=None): + """ + Get all signing certificates associated with an account. + + If the user_name is not specified, it is determined implicitly based + on the AWS Access Key ID used to sign the request. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + + :type user_name: string + :param user_name: The username of the user + + """ + params = {} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + if user_name: + params['UserName'] = user_name + return self.get_response('ListSigningCertificates', + params, list_marker='Certificates') + + def update_signing_cert(self, cert_id, status, user_name=None): + """ + Change the status of the specified signing certificate from + Active to Inactive or vice versa. + + If the user_name is not specified, it is determined implicitly based + on the AWS Access Key ID used to sign the request. + + :type cert_id: string + :param cert_id: The ID of the signing certificate + + :type status: string + :param status: Either Active or Inactive. + + :type user_name: string + :param user_name: The username of the user + """ + params = {'CertificateId' : cert_id, + 'Status' : status} + if user_name: + params['UserName'] = user_name + return self.get_response('UpdateSigningCertificate', params) + + def upload_signing_cert(self, cert_body, user_name=None): + """ + Uploads an X.509 signing certificate and associates it with + the specified user. + + If the user_name is not specified, it is determined implicitly based + on the AWS Access Key ID used to sign the request. + + :type cert_body: string + :param cert_body: The body of the signing certificate. + + :type user_name: string + :param user_name: The username of the new user + + """ + params = {'CertificateBody' : cert_body} + if user_name: + params['UserName'] = user_name + return self.get_response('UploadSigningCertificate', params, + verb='POST') + + def delete_signing_cert(self, cert_id, user_name=None): + """ + Delete a signing certificate associated with a user. + + If the user_name is not specified, it is determined implicitly based + on the AWS Access Key ID used to sign the request. + + :type user_name: string + :param user_name: The username of the new user + + :type cert_id: string + :param cert_id: The ID of the certificate. + + """ + params = {'CertificateId' : cert_id} + if user_name: + params['UserName'] = user_name + return self.get_response('DeleteSigningCertificate', params) + + # + # Server Certificates + # + + def get_all_server_certs(self, path_prefix='/', + marker=None, max_items=None): + """ + Lists the server certificates that have the specified path prefix. + If none exist, the action returns an empty list. + + :type path_prefix: string + :param path_prefix: If provided, only certificates whose paths match + the provided prefix will be returned. + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + + """ + params = {} + if path_prefix: + params['PathPrefix'] = path_prefix + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListServerCertificates', + params, + list_marker='ServerCertificateMetadataList') + + def update_server_cert(self, cert_name, new_cert_name=None, + new_path=None): + """ + Updates the name and/or the path of the specified server certificate. + + :type cert_name: string + :param cert_name: The name of the server certificate that you want + to update. + + :type new_cert_name: string + :param new_cert_name: The new name for the server certificate. + Include this only if you are updating the + server certificate's name. + + :type new_path: string + :param new_path: If provided, the path of the certificate will be + changed to this path. + """ + params = {'ServerCertificateName' : cert_name} + if new_cert_name: + params['NewServerCertificateName'] = new_cert_name + if new_path: + params['NewPath'] = new_path + return self.get_response('UpdateServerCertificate', params) + + def upload_server_cert(self, cert_name, cert_body, private_key, + cert_chain=None, path=None): + """ + Uploads a server certificate entity for the AWS Account. + The server certificate entity includes a public key certificate, + a private key, and an optional certificate chain, which should + all be PEM-encoded. + + :type cert_name: string + :param cert_name: The name for the server certificate. Do not + include the path in this value. + + :type cert_body: string + :param cert_body: The contents of the public key certificate + in PEM-encoded format. + + :type private_key: string + :param private_key: The contents of the private key in + PEM-encoded format. + + :type cert_chain: string + :param cert_chain: The contents of the certificate chain. This + is typically a concatenation of the PEM-encoded + public key certificates of the chain. + + :type path: string + :param path: The path for the server certificate. + + """ + params = {'ServerCertificateName' : cert_name, + 'CertificateBody' : cert_body, + 'PrivateKey' : private_key} + if cert_chain: + params['CertificateChain'] = cert_chain + if path: + params['Path'] = path + return self.get_response('UploadServerCertificate', params, + verb='POST') + + def get_server_certificate(self, cert_name): + """ + Retrieves information about the specified server certificate. + + :type cert_name: string + :param cert_name: The name of the server certificate you want + to retrieve information about. + + """ + params = {'ServerCertificateName' : cert_name} + return self.get_response('GetServerCertificate', params) + + def delete_server_cert(self, cert_name): + """ + Delete the specified server certificate. + + :type cert_name: string + :param cert_name: The name of the server certificate you want + to delete. + + """ + params = {'ServerCertificateName' : cert_name} + return self.get_response('DeleteServerCertificate', params) + + # + # MFA Devices + # + + def get_all_mfa_devices(self, user_name, marker=None, max_items=None): + """ + Get all MFA devices associated with an account. + + :type user_name: string + :param user_name: The username of the user + + :type marker: string + :param marker: Use this only when paginating results and only in + follow-up request after you've received a response + where the results are truncated. Set this to the + value of the Marker element in the response you + just received. + + :type max_items: int + :param max_items: Use this only when paginating results to indicate + the maximum number of groups you want in the + response. + + """ + params = {'UserName' : user_name} + if marker: + params['Marker'] = marker + if max_items: + params['MaxItems'] = max_items + return self.get_response('ListMFADevices', + params, list_marker='MFADevices') + + def enable_mfa_device(self, user_name, serial_number, + auth_code_1, auth_code_2): + """ + Enables the specified MFA device and associates it with the + specified user. + + :type user_name: string + :param user_name: The username of the user + + :type serial_number: string + :param seriasl_number: The serial number which uniquely identifies + the MFA device. + + :type auth_code_1: string + :param auth_code_1: An authentication code emitted by the device. + + :type auth_code_2: string + :param auth_code_2: A subsequent authentication code emitted + by the device. + + """ + params = {'UserName' : user_name, + 'SerialNumber' : serial_number, + 'AuthenticationCode1' : auth_code_1, + 'AuthenticationCode2' : auth_code_2} + return self.get_response('EnableMFADevice', params) + + def deactivate_mfa_device(self, user_name, serial_number): + """ + Deactivates the specified MFA device and removes it from + association with the user. + + :type user_name: string + :param user_name: The username of the user + + :type serial_number: string + :param seriasl_number: The serial number which uniquely identifies + the MFA device. + + """ + params = {'UserName' : user_name, + 'SerialNumber' : serial_number} + return self.get_response('DeactivateMFADevice', params) + + def resync_mfa_device(self, user_name, serial_number, + auth_code_1, auth_code_2): + """ + Syncronizes the specified MFA device with the AWS servers. + + :type user_name: string + :param user_name: The username of the user + + :type serial_number: string + :param seriasl_number: The serial number which uniquely identifies + the MFA device. + + :type auth_code_1: string + :param auth_code_1: An authentication code emitted by the device. + + :type auth_code_2: string + :param auth_code_2: A subsequent authentication code emitted + by the device. + + """ + params = {'UserName' : user_name, + 'SerialNumber' : serial_number, + 'AuthenticationCode1' : auth_code_1, + 'AuthenticationCode2' : auth_code_2} + return self.get_response('ResyncMFADevice', params) + + # + # Login Profiles + # + + def create_login_profile(self, user_name, password): + """ + Creates a login profile for the specified user, give the user the + ability to access AWS services and the AWS Management Console. + + :type user_name: string + :param user_name: The name of the new user + + :type password: string + :param password: The new password for the user + + """ + params = {'UserName' : user_name, + 'Password' : password} + return self.get_response('CreateLoginProfile', params) + + def delete_login_profile(self, user_name): + """ + Deletes the login profile associated with the specified user. + + :type user_name: string + :param user_name: The name of the user to delete. + + """ + params = {'UserName' : user_name} + return self.get_response('DeleteLoginProfile', params) + + def update_login_profile(self, user_name, password): + """ + Resets the password associated with the user's login profile. + + :type user_name: string + :param user_name: The name of the user + + :type password: string + :param password: The new password for the user + + """ + params = {'UserName' : user_name, + 'Password' : password} + return self.get_response('UpdateLoginProfile', params) + + def create_account_alias(self, alias): + """ + Creates a new alias for the AWS account. + + For more information on account id aliases, please see + http://goo.gl/ToB7G + + :type alias: string + :param alias: The alias to attach to the account. + """ + params = {'AccountAlias': alias} + return self.get_response('CreateAccountAlias', params) + + def delete_account_alias(self, alias): + """ + Deletes an alias for the AWS account. + + For more information on account id aliases, please see + http://goo.gl/ToB7G + + :type alias: string + :param alias: The alias to remove from the account. + """ + params = {'AccountAlias': alias} + return self.get_response('DeleteAccountAlias', params) + + def get_account_alias(self): + """ + Get the alias for the current account. + + This is referred to in the docs as list_account_aliases, + but it seems you can only have one account alias currently. + + For more information on account id aliases, please see + http://goo.gl/ToB7G + """ + r = self.get_response('ListAccountAliases', {}) + response = r.get('ListAccountAliasesResponse') + result = response.get('ListAccountAliasesResult') + aliases = result.get('AccountAliases') + return aliases.get('member', None) + + def get_signin_url(self, service='ec2'): + """ + Get the URL where IAM users can use their login profile to sign in + to this account's console. + + :type service: string + :param service: Default service to go to in the console. + """ + alias = self.get_account_alias() + if not alias: + raise Exception('No alias associated with this account. Please use iam.create_account_alias() first.') + + return "https://%s.signin.aws.amazon.com/console/%s" % (alias, service) diff --git a/backup/src/boto/jsonresponse.py b/backup/src/boto/jsonresponse.py new file mode 100644 index 0000000..beb50ce --- /dev/null +++ b/backup/src/boto/jsonresponse.py @@ -0,0 +1,143 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import utils + +class XmlHandler(xml.sax.ContentHandler): + + def __init__(self, root_node, connection): + self.connection = connection + self.nodes = [('root', root_node)] + self.current_text = '' + + def startElement(self, name, attrs): + self.current_text = '' + t = self.nodes[-1][1].startElement(name, attrs, self.connection) + if t != None: + if isinstance(t, tuple): + self.nodes.append(t) + else: + self.nodes.append((name, t)) + + def endElement(self, name): + self.nodes[-1][1].endElement(name, self.current_text, self.connection) + if self.nodes[-1][0] == name: + self.nodes.pop() + self.current_text = '' + + def characters(self, content): + self.current_text += content + + def parse(self, s): + xml.sax.parseString(s, self) + +class Element(dict): + + def __init__(self, connection=None, element_name=None, + stack=None, parent=None, list_marker=('Set',), + item_marker=('member', 'item')): + dict.__init__(self) + self.connection = connection + self.element_name = element_name + self.list_marker = utils.mklist(list_marker) + self.item_marker = utils.mklist(item_marker) + if stack is None: + self.stack = [] + else: + self.stack = stack + self.parent = parent + + def __getattr__(self, key): + if key in self: + return self[key] + for k in self: + e = self[k] + if isinstance(e, Element): + try: + return getattr(e, key) + except AttributeError: + pass + raise AttributeError + + def startElement(self, name, attrs, connection): + self.stack.append(name) + for lm in self.list_marker: + if name.endswith(lm): + l = ListElement(self.connection, name, self.list_marker, + self.item_marker) + self[name] = l + return l + if len(self.stack) > 0: + element_name = self.stack[-1] + e = Element(self.connection, element_name, self.stack, self, + self.list_marker, self.item_marker) + self[element_name] = e + return (element_name, e) + else: + return None + + def endElement(self, name, value, connection): + if len(self.stack) > 0: + self.stack.pop() + value = value.strip() + if value: + if isinstance(self.parent, Element): + self.parent[name] = value + elif isinstance(self.parent, ListElement): + self.parent.append(value) + +class ListElement(list): + + def __init__(self, connection=None, element_name=None, + list_marker=['Set'], item_marker=('member', 'item')): + list.__init__(self) + self.connection = connection + self.element_name = element_name + self.list_marker = list_marker + self.item_marker = item_marker + + def startElement(self, name, attrs, connection): + for lm in self.list_marker: + if name.endswith(lm): + l = ListElement(self.connection, name, self.item_marker) + setattr(self, name, l) + return l + if name in self.item_marker: + e = Element(self.connection, name, parent=self) + self.append(e) + return e + else: + return None + + def endElement(self, name, value, connection): + if name == self.element_name: + if len(self) > 0: + empty = [] + for e in self: + if isinstance(e, Element): + if len(e) == 0: + empty.append(e) + for e in empty: + self.remove(e) + else: + setattr(self, name, value) diff --git a/backup/src/boto/manage/__init__.py b/backup/src/boto/manage/__init__.py new file mode 100644 index 0000000..49d029b --- /dev/null +++ b/backup/src/boto/manage/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/backup/src/boto/manage/cmdshell.py b/backup/src/boto/manage/cmdshell.py new file mode 100644 index 0000000..cbd2e60 --- /dev/null +++ b/backup/src/boto/manage/cmdshell.py @@ -0,0 +1,174 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.mashups.interactive import interactive_shell +import boto +import os +import time +import shutil +import StringIO +import paramiko +import socket +import subprocess + + +class SSHClient(object): + + def __init__(self, server, + host_key_file='~/.ssh/known_hosts', + uname='root', ssh_pwd=None): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + self._pkey = paramiko.RSAKey.from_private_key_file(server.ssh_key_file, + password=ssh_pwd) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.connect() + + def connect(self): + retry = 0 + while retry < 5: + try: + self._ssh_client.connect(self.server.hostname, + username=self.uname, + pkey=self._pkey) + return + except socket.error, (value,message): + if value == 61 or value == 111: + print 'SSH Connection refused, will retry in 5 seconds' + time.sleep(5) + retry += 1 + else: + raise + except paramiko.BadHostKeyException: + print "%s has an entry in ~/.ssh/known_hosts and it doesn't match" % self.server.hostname + print 'Edit that file to remove the entry and then hit return to try again' + raw_input('Hit Enter when ready') + retry += 1 + except EOFError: + print 'Unexpected Error from SSH Connection, retry in 5 seconds' + time.sleep(5) + retry += 1 + print 'Could not establish SSH connection' + + def get_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.get(src, dst) + + def put_file(self, src, dst): + sftp_client = self._ssh_client.open_sftp() + sftp_client.put(src, dst) + + def listdir(self, path): + sftp_client = self._ssh_client.open_sftp() + return sftp_client.listdir(path) + + def open_sftp(self): + return self._ssh_client.open_sftp() + + def isdir(self, path): + status = self.run('[ -d %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def exists(self, path): + status = self.run('[ -a %s ] || echo "FALSE"' % path) + if status[1].startswith('FALSE'): + return 0 + return 1 + + def shell(self): + channel = self._ssh_client.invoke_shell() + interactive_shell(channel) + + def run(self, command): + boto.log.info('running:%s on %s' % (command, self.server.instance_id)) + log_fp = StringIO.StringIO() + status = 0 + try: + t = self._ssh_client.exec_command(command) + except paramiko.SSHException: + status = 1 + log_fp.write(t[1].read()) + log_fp.write(t[2].read()) + t[0].close() + t[1].close() + t[2].close() + boto.log.info('output: %s' % log_fp.getvalue()) + return (status, log_fp.getvalue()) + + def close(self): + transport = self._ssh_client.get_transport() + transport.close() + self.server.reset_cmdshell() + +class LocalClient(object): + + def __init__(self, server, host_key_file=None, uname='root'): + self.server = server + self.host_key_file = host_key_file + self.uname = uname + + def get_file(self, src, dst): + shutil.copyfile(src, dst) + + def put_file(self, src, dst): + shutil.copyfile(src, dst) + + def listdir(self, path): + return os.listdir(path) + + def isdir(self, path): + return os.path.isdir(path) + + def exists(self, path): + return os.path.exists(path) + + def shell(self): + raise NotImplementedError, 'shell not supported with LocalClient' + + def run(self): + boto.log.info('running:%s' % self.command) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while process.poll() == None: + time.sleep(1) + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info(log_fp.getvalue()) + boto.log.info('output: %s' % log_fp.getvalue()) + return (process.returncode, log_fp.getvalue()) + + def close(self): + pass + +def start(server): + instance_id = boto.config.get('Instance', 'instance-id', None) + if instance_id == server.instance_id: + return LocalClient(server) + else: + return SSHClient(server) diff --git a/backup/src/boto/manage/propget.py b/backup/src/boto/manage/propget.py new file mode 100644 index 0000000..45b2ff2 --- /dev/null +++ b/backup/src/boto/manage/propget.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +def get(prop, choices=None): + prompt = prop.verbose_name + if not prompt: + prompt = prop.name + if choices: + if callable(choices): + choices = choices() + else: + choices = prop.get_choices() + valid = False + while not valid: + if choices: + min = 1 + max = len(choices) + for i in range(min, max+1): + value = choices[i-1] + if isinstance(value, tuple): + value = value[0] + print '[%d] %s' % (i, value) + value = raw_input('%s [%d-%d]: ' % (prompt, min, max)) + try: + int_value = int(value) + value = choices[int_value-1] + if isinstance(value, tuple): + value = value[1] + valid = True + except ValueError: + print '%s is not a valid choice' % value + except IndexError: + print '%s is not within the range[%d-%d]' % (min, max) + else: + value = raw_input('%s: ' % prompt) + try: + value = prop.validate(value) + if prop.empty(value) and prop.required: + print 'A value is required' + else: + valid = True + except: + print 'Invalid value: %s' % value + return value + diff --git a/backup/src/boto/manage/server.py b/backup/src/boto/manage/server.py new file mode 100644 index 0000000..3c7a303 --- /dev/null +++ b/backup/src/boto/manage/server.py @@ -0,0 +1,556 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +from __future__ import with_statement +import boto.ec2 +from boto.mashups.iobject import IObject +from boto.pyami.config import BotoConfigPath, Config +from boto.sdb.db.model import Model +from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty, CalculatedProperty +from boto.manage import propget +from boto.ec2.zone import Zone +from boto.ec2.keypair import KeyPair +import os, time, StringIO +from contextlib import closing +from boto.exception import EC2ResponseError + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', + 'c1.medium', 'c1.xlarge', + 'm2.2xlarge', 'm2.4xlarge'] + +class Bundler(object): + + def __init__(self, server, uname='root'): + from boto.manage.cmdshell import SSHClient + self.server = server + self.uname = uname + self.ssh_client = SSHClient(server, uname=uname) + + def copy_x509(self, key_file, cert_file): + print '\tcopying cert and pk over to /mnt directory on server' + self.ssh_client.open_sftp() + path, name = os.path.split(key_file) + self.remote_key_file = '/mnt/%s' % name + self.ssh_client.put_file(key_file, self.remote_key_file) + path, name = os.path.split(cert_file) + self.remote_cert_file = '/mnt/%s' % name + self.ssh_client.put_file(cert_file, self.remote_cert_file) + print '...complete!' + + def bundle_image(self, prefix, size, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-bundle-vol ' + command += '-c %s -k %s ' % (self.remote_cert_file, self.remote_key_file) + command += '-u %s ' % self.server._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.server.instance_type == 'm1.small' or self.server.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + return command + + def upload_bundle(self, bucket, prefix, ssh_key): + command = "" + if self.uname != 'root': + command = "sudo " + command += 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.server.ec2.aws_access_key_id + command += '-s %s ' % self.server.ec2.aws_secret_access_key + return command + + def bundle(self, bucket=None, prefix=None, key_file=None, cert_file=None, + size=None, ssh_key=None, fp=None, clear_history=True): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + if not ssh_key: + ssh_key = self.server.get_ssh_key_file() + self.copy_x509(key_file, cert_file) + if not fp: + fp = StringIO.StringIO() + fp.write('sudo mv %s /mnt/boto.cfg; ' % BotoConfigPath) + fp.write('mv ~/.ssh/authorized_keys /mnt/authorized_keys; ') + if clear_history: + fp.write('history -c; ') + fp.write(self.bundle_image(prefix, size, ssh_key)) + fp.write('; ') + fp.write(self.upload_bundle(bucket, prefix, ssh_key)) + fp.write('; ') + fp.write('sudo mv /mnt/boto.cfg %s; ' % BotoConfigPath) + fp.write('mv /mnt/authorized_keys ~/.ssh/authorized_keys') + command = fp.getvalue() + print 'running the following command on the remote server:' + print command + t = self.ssh_client.run(command) + print '\t%s' % t[0] + print '\t%s' % t[1] + print '...complete!' + print 'registering image...' + self.image_id = self.server.ec2.register_image(name=prefix, image_location='%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + +class CommandLineGetter(object): + + def get_ami_list(self): + my_amis = [] + for ami in self.ec2.get_all_images(): + # hack alert, need a better way to do this! + if ami.location.find('pyami') >= 0: + my_amis.append((ami.location, ami)) + return my_amis + + def get_region(self, params): + region = params.get('region', None) + if isinstance(region, str) or isinstance(region, unicode): + region = boto.ec2.get_region(region) + params['region'] = region + if not region: + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + self.ec2 = params['region'].connect() + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_description(self, params): + if not params.get('description', None): + prop = self.cls.find_property('description') + params['description'] = propget.get(prop) + + def get_instance_type(self, params): + if not params.get('instance_type', None): + prop = StringProperty(name='instance_type', verbose_name='Instance Type', + choices=InstanceTypes) + params['instance_type'] = propget.get(prop) + + def get_quantity(self, params): + if not params.get('quantity', None): + prop = IntegerProperty(name='quantity', verbose_name='Number of Instances') + params['quantity'] = propget.get(prop) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_ami_id(self, params): + valid = False + while not valid: + ami = params.get('ami', None) + if not ami: + prop = StringProperty(name='ami', verbose_name='AMI') + ami = propget.get(prop) + try: + rs = self.ec2.get_all_images([ami]) + if len(rs) == 1: + valid = True + params['ami'] = rs[0] + except EC2ResponseError: + pass + + def get_group(self, params): + group = params.get('group', None) + if isinstance(group, str) or isinstance(group, unicode): + group_list = self.ec2.get_all_security_groups() + for g in group_list: + if g.name == group: + group = g + params['group'] = g + if not group: + prop = StringProperty(name='group', verbose_name='EC2 Security Group', + choices=self.ec2.get_all_security_groups) + params['group'] = propget.get(prop) + + def get_key(self, params): + keypair = params.get('keypair', None) + if isinstance(keypair, str) or isinstance(keypair, unicode): + key_list = self.ec2.get_all_key_pairs() + for k in key_list: + if k.name == keypair: + keypair = k.name + params['keypair'] = k.name + if not keypair: + prop = StringProperty(name='keypair', verbose_name='EC2 KeyPair', + choices=self.ec2.get_all_key_pairs) + params['keypair'] = propget.get(prop).name + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_name(params) + self.get_description(params) + self.get_instance_type(params) + self.get_zone(params) + self.get_quantity(params) + self.get_ami_id(params) + self.get_group(params) + self.get_key(params) + +class Server(Model): + + # + # The properties of this object consists of real properties for data that + # is not already stored in EC2 somewhere (e.g. name, description) plus + # calculated properties for all of the properties that are already in + # EC2 (e.g. hostname, security groups, etc.) + # + name = StringProperty(unique=True, verbose_name="Name") + description = StringProperty(verbose_name="Description") + region_name = StringProperty(verbose_name="EC2 Region Name") + instance_id = StringProperty(verbose_name="EC2 Instance ID") + elastic_ip = StringProperty(verbose_name="EC2 Elastic IP Address") + production = BooleanProperty(verbose_name="Is This Server Production", default=False) + ami_id = CalculatedProperty(verbose_name="AMI ID", calculated_type=str, use_method=True) + zone = CalculatedProperty(verbose_name="Availability Zone Name", calculated_type=str, use_method=True) + hostname = CalculatedProperty(verbose_name="Public DNS Name", calculated_type=str, use_method=True) + private_hostname = CalculatedProperty(verbose_name="Private DNS Name", calculated_type=str, use_method=True) + groups = CalculatedProperty(verbose_name="Security Groups", calculated_type=list, use_method=True) + security_group = CalculatedProperty(verbose_name="Primary Security Group Name", calculated_type=str, use_method=True) + key_name = CalculatedProperty(verbose_name="Key Name", calculated_type=str, use_method=True) + instance_type = CalculatedProperty(verbose_name="Instance Type", calculated_type=str, use_method=True) + status = CalculatedProperty(verbose_name="Current Status", calculated_type=str, use_method=True) + launch_time = CalculatedProperty(verbose_name="Server Launch Time", calculated_type=str, use_method=True) + console_output = CalculatedProperty(verbose_name="Console Output", calculated_type=file, use_method=True) + + packages = [] + plugins = [] + + @classmethod + def add_credentials(cls, cfg, aws_access_key_id, aws_secret_access_key): + if not cfg.has_section('Credentials'): + cfg.add_section('Credentials') + cfg.set('Credentials', 'aws_access_key_id', aws_access_key_id) + cfg.set('Credentials', 'aws_secret_access_key', aws_secret_access_key) + if not cfg.has_section('DB_Server'): + cfg.add_section('DB_Server') + cfg.set('DB_Server', 'db_type', 'SimpleDB') + cfg.set('DB_Server', 'db_name', cls._manager.domain.name) + + @classmethod + def create(cls, config_file=None, logical_volume = None, cfg = None, **params): + """ + Create a new instance based on the specified configuration file or the specified + configuration and the passed in parameters. + + If the config_file argument is not None, the configuration is read from there. + Otherwise, the cfg argument is used. + + The config file may include other config files with a #import reference. The included + config files must reside in the same directory as the specified file. + + The logical_volume argument, if supplied, will be used to get the current physical + volume ID and use that as an override of the value specified in the config file. This + may be useful for debugging purposes when you want to debug with a production config + file but a test Volume. + + The dictionary argument may be used to override any EC2 configuration values in the + config file. + """ + if config_file: + cfg = Config(path=config_file) + if cfg.has_section('EC2'): + # include any EC2 configuration values that aren't specified in params: + for option in cfg.options('EC2'): + if option not in params: + params[option] = cfg.get('EC2', option) + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + cls.add_credentials(cfg, ec2.aws_access_key_id, ec2.aws_secret_access_key) + ami = params.get('ami') + kp = params.get('keypair') + group = params.get('group') + zone = params.get('zone') + # deal with possibly passed in logical volume: + if logical_volume != None: + cfg.set('EBS', 'logical_volume_name', logical_volume.name) + cfg_fp = StringIO.StringIO() + cfg.write(cfg_fp) + # deal with the possibility that zone and/or keypair are strings read from the config file: + if isinstance(zone, Zone): + zone = zone.name + if isinstance(kp, KeyPair): + kp = kp.name + reservation = ami.run(min_count=1, + max_count=params.get('quantity', 1), + key_name=kp, + security_groups=[group], + instance_type=params.get('instance_type'), + placement = zone, + user_data = cfg_fp.getvalue()) + l = [] + i = 0 + elastic_ip = params.get('elastic_ip') + instances = reservation.instances + if elastic_ip != None and instances.__len__() > 0: + instance = instances[0] + print 'Waiting for instance to start so we can set its elastic IP address...' + # Sometimes we get a message from ec2 that says that the instance does not exist. + # Hopefully the following delay will giv eec2 enough time to get to a stable state: + time.sleep(5) + while instance.update() != 'running': + time.sleep(1) + instance.use_ip(elastic_ip) + print 'set the elastic IP of the first instance to %s' % elastic_ip + for instance in instances: + s = cls() + s.ec2 = ec2 + s.name = params.get('name') + '' if i==0 else str(i) + s.description = params.get('description') + s.region_name = region.name + s.instance_id = instance.id + if elastic_ip and i == 0: + s.elastic_ip = elastic_ip + s.put() + l.append(s) + i += 1 + return l + + @classmethod + def create_from_instance_id(cls, instance_id, name, description=''): + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + try: + rs = ec2.get_all_instances([instance_id]) + except: + rs = [] + if len(rs) == 1: + s = cls() + s.ec2 = ec2 + s.name = name + s.description = description + s.region_name = region.name + s.instance_id = instance_id + s._reservation = rs[0] + for instance in s._reservation.instances: + if instance.id == instance_id: + s._instance = instance + s.put() + return s + return None + + @classmethod + def create_from_current_instances(cls): + servers = [] + regions = boto.ec2.regions() + for region in regions: + ec2 = region.connect() + rs = ec2.get_all_instances() + for reservation in rs: + for instance in reservation.instances: + try: + Server.find(instance_id=instance.id).next() + boto.log.info('Server for %s already exists' % instance.id) + except StopIteration: + s = cls() + s.ec2 = ec2 + s.name = instance.id + s.region_name = region.name + s.instance_id = instance.id + s._reservation = reservation + s.put() + servers.append(s) + return servers + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.ssh_key_file = None + self.ec2 = None + self._cmdshell = None + self._reservation = None + self._instance = None + self._setup_ec2() + + def _setup_ec2(self): + if self.ec2 and self._instance and self._reservation: + return + if self.id: + if self.region_name: + for region in boto.ec2.regions(): + if region.name == self.region_name: + self.ec2 = region.connect() + if self.instance_id and not self._instance: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + if len(rs) >= 1: + for instance in rs[0].instances: + if instance.id == self.instance_id: + self._reservation = rs[0] + self._instance = instance + except EC2ResponseError: + pass + + def _status(self): + status = '' + if self._instance: + self._instance.update() + status = self._instance.state + return status + + def _hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.public_dns_name + return hostname + + def _private_hostname(self): + hostname = '' + if self._instance: + hostname = self._instance.private_dns_name + return hostname + + def _instance_type(self): + it = '' + if self._instance: + it = self._instance.instance_type + return it + + def _launch_time(self): + lt = '' + if self._instance: + lt = self._instance.launch_time + return lt + + def _console_output(self): + co = '' + if self._instance: + co = self._instance.get_console_output() + return co + + def _groups(self): + gn = [] + if self._reservation: + gn = self._reservation.groups + return gn + + def _security_group(self): + groups = self._groups() + if len(groups) >= 1: + return groups[0].id + return "" + + def _zone(self): + zone = None + if self._instance: + zone = self._instance.placement + return zone + + def _key_name(self): + kn = None + if self._instance: + kn = self._instance.key_name + return kn + + def put(self): + Model.put(self) + self._setup_ec2() + + def delete(self): + if self.production: + raise ValueError, "Can't delete a production server" + #self.stop() + Model.delete(self) + + def stop(self): + if self.production: + raise ValueError, "Can't delete a production server" + if self._instance: + self._instance.stop() + + def terminate(self): + if self.production: + raise ValueError, "Can't delete a production server" + if self._instance: + self._instance.terminate() + + def reboot(self): + if self._instance: + self._instance.reboot() + + def wait(self): + while self.status != 'running': + time.sleep(5) + + def get_ssh_key_file(self): + if not self.ssh_key_file: + ssh_dir = os.path.expanduser('~/.ssh') + if os.path.isdir(ssh_dir): + ssh_file = os.path.join(ssh_dir, '%s.pem' % self.key_name) + if os.path.isfile(ssh_file): + self.ssh_key_file = ssh_file + if not self.ssh_key_file: + iobject = IObject() + self.ssh_key_file = iobject.get_filename('Path to OpenSSH Key file') + return self.ssh_key_file + + def get_cmdshell(self): + if not self._cmdshell: + import cmdshell + self.get_ssh_key_file() + self._cmdshell = cmdshell.start(self) + return self._cmdshell + + def reset_cmdshell(self): + self._cmdshell = None + + def run(self, command): + with closing(self.get_cmdshell()) as cmd: + status = cmd.run(command) + return status + + def get_bundler(self, uname='root'): + self.get_ssh_key_file() + return Bundler(self, uname) + + def get_ssh_client(self, uname='root', ssh_pwd=None): + from boto.manage.cmdshell import SSHClient + self.get_ssh_key_file() + return SSHClient(self, uname=uname, ssh_pwd=ssh_pwd) + + def install(self, pkg): + return self.run('apt-get -y install %s' % pkg) + + + diff --git a/backup/src/boto/manage/task.py b/backup/src/boto/manage/task.py new file mode 100644 index 0000000..2f9d7d0 --- /dev/null +++ b/backup/src/boto/manage/task.py @@ -0,0 +1,175 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto +from boto.sdb.db.property import StringProperty, DateTimeProperty, IntegerProperty +from boto.sdb.db.model import Model +import datetime, subprocess, StringIO, time + +def check_hour(val): + if val == '*': + return + if int(val) < 0 or int(val) > 23: + raise ValueError + +class Task(Model): + + """ + A scheduled, repeating task that can be executed by any participating servers. + The scheduling is similar to cron jobs. Each task has an hour attribute. + The allowable values for hour are [0-23|*]. + + To keep the operation reasonably efficient and not cause excessive polling, + the minimum granularity of a Task is hourly. Some examples: + + hour='*' - the task would be executed each hour + hour='3' - the task would be executed at 3AM GMT each day. + + """ + name = StringProperty() + hour = StringProperty(required=True, validator=check_hour, default='*') + command = StringProperty(required=True) + last_executed = DateTimeProperty() + last_status = IntegerProperty() + last_output = StringProperty() + message_id = StringProperty() + + @classmethod + def start_all(cls, queue_name): + for task in cls.all(): + task.start(queue_name) + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self.hourly = self.hour == '*' + self.daily = self.hour != '*' + self.now = datetime.datetime.utcnow() + + def check(self): + """ + Determine how long until the next scheduled time for a Task. + Returns the number of seconds until the next scheduled time or zero + if the task needs to be run immediately. + If it's an hourly task and it's never been run, run it now. + If it's a daily task and it's never been run and the hour is right, run it now. + """ + boto.log.info('checking Task[%s]-now=%s, last=%s' % (self.name, self.now, self.last_executed)) + + if self.hourly and not self.last_executed: + return 0 + + if self.daily and not self.last_executed: + if int(self.hour) == self.now.hour: + return 0 + else: + return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 + + delta = self.now - self.last_executed + if self.hourly: + if delta.seconds >= 60*60: + return 0 + else: + return 60*60 - delta.seconds + else: + if int(self.hour) == self.now.hour: + if delta.days >= 1: + return 0 + else: + return 82800 # 23 hours, just to be safe + else: + return max( (int(self.hour)-self.now.hour), (self.now.hour-int(self.hour)) )*60*60 + + def _run(self, msg, vtimeout): + boto.log.info('Task[%s] - running:%s' % (self.name, self.command)) + log_fp = StringIO.StringIO() + process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + nsecs = 5 + current_timeout = vtimeout + while process.poll() == None: + boto.log.info('nsecs=%s, timeout=%s' % (nsecs, current_timeout)) + if nsecs >= current_timeout: + current_timeout += vtimeout + boto.log.info('Task[%s] - setting timeout to %d seconds' % (self.name, current_timeout)) + if msg: + msg.change_visibility(current_timeout) + time.sleep(5) + nsecs += 5 + t = process.communicate() + log_fp.write(t[0]) + log_fp.write(t[1]) + boto.log.info('Task[%s] - output: %s' % (self.name, log_fp.getvalue())) + self.last_executed = self.now + self.last_status = process.returncode + self.last_output = log_fp.getvalue()[0:1023] + + def run(self, msg, vtimeout=60): + delay = self.check() + boto.log.info('Task[%s] - delay=%s seconds' % (self.name, delay)) + if delay == 0: + self._run(msg, vtimeout) + queue = msg.queue + new_msg = queue.new_message(self.id) + new_msg = queue.write(new_msg) + self.message_id = new_msg.id + self.put() + boto.log.info('Task[%s] - new message id=%s' % (self.name, new_msg.id)) + msg.delete() + boto.log.info('Task[%s] - deleted message %s' % (self.name, msg.id)) + else: + boto.log.info('new_vtimeout: %d' % delay) + msg.change_visibility(delay) + + def start(self, queue_name): + boto.log.info('Task[%s] - starting with queue: %s' % (self.name, queue_name)) + queue = boto.lookup('sqs', queue_name) + msg = queue.new_message(self.id) + msg = queue.write(msg) + self.message_id = msg.id + self.put() + boto.log.info('Task[%s] - start successful' % self.name) + +class TaskPoller(object): + + def __init__(self, queue_name): + self.sqs = boto.connect_sqs() + self.queue = self.sqs.lookup(queue_name) + + def poll(self, wait=60, vtimeout=60): + while 1: + m = self.queue.read(vtimeout) + if m: + task = Task.get_by_id(m.get_body()) + if task: + if not task.message_id or m.id == task.message_id: + boto.log.info('Task[%s] - read message %s' % (task.name, m.id)) + task.run(m, vtimeout) + else: + boto.log.info('Task[%s] - found extraneous message, ignoring' % task.name) + else: + time.sleep(wait) + + + + + + diff --git a/backup/src/boto/manage/test_manage.py b/backup/src/boto/manage/test_manage.py new file mode 100644 index 0000000..e0b032a --- /dev/null +++ b/backup/src/boto/manage/test_manage.py @@ -0,0 +1,34 @@ +from boto.manage.server import Server +from boto.manage.volume import Volume +import time + +print '--> Creating New Volume' +volume = Volume.create() +print volume + +print '--> Creating New Server' +server_list = Server.create() +server = server_list[0] +print server + +print '----> Waiting for Server to start up' +while server.status != 'running': + print '*' + time.sleep(10) +print '----> Server is running' + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Now run volume.make_ready to make the volume ready to use on server' +volume.make_ready(server) + +print '--> Run "df -k" on Server' +status = server.run('df -k') +print status[1] + +print '--> Do an "ls -al" on the new filesystem' +status = server.run('ls -al %s' % volume.mount_point) +print status[1] + diff --git a/backup/src/boto/manage/volume.py b/backup/src/boto/manage/volume.py new file mode 100644 index 0000000..66a458f --- /dev/null +++ b/backup/src/boto/manage/volume.py @@ -0,0 +1,420 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from __future__ import with_statement +from boto.sdb.db.model import Model +from boto.sdb.db.property import StringProperty, IntegerProperty, ListProperty, ReferenceProperty, CalculatedProperty +from boto.manage.server import Server +from boto.manage import propget +import boto.ec2 +import time +import traceback +from contextlib import closing +import dateutil.parser +import datetime + + +class CommandLineGetter(object): + + def get_region(self, params): + if not params.get('region', None): + prop = self.cls.find_property('region_name') + params['region'] = propget.get(prop, choices=boto.ec2.regions) + + def get_zone(self, params): + if not params.get('zone', None): + prop = StringProperty(name='zone', verbose_name='EC2 Availability Zone', + choices=self.ec2.get_all_zones) + params['zone'] = propget.get(prop) + + def get_name(self, params): + if not params.get('name', None): + prop = self.cls.find_property('name') + params['name'] = propget.get(prop) + + def get_size(self, params): + if not params.get('size', None): + prop = IntegerProperty(name='size', verbose_name='Size (GB)') + params['size'] = propget.get(prop) + + def get_mount_point(self, params): + if not params.get('mount_point', None): + prop = self.cls.find_property('mount_point') + params['mount_point'] = propget.get(prop) + + def get_device(self, params): + if not params.get('device', None): + prop = self.cls.find_property('device') + params['device'] = propget.get(prop) + + def get(self, cls, params): + self.cls = cls + self.get_region(params) + self.ec2 = params['region'].connect() + self.get_zone(params) + self.get_name(params) + self.get_size(params) + self.get_mount_point(params) + self.get_device(params) + +class Volume(Model): + + name = StringProperty(required=True, unique=True, verbose_name='Name') + region_name = StringProperty(required=True, verbose_name='EC2 Region') + zone_name = StringProperty(required=True, verbose_name='EC2 Zone') + mount_point = StringProperty(verbose_name='Mount Point') + device = StringProperty(verbose_name="Device Name", default='/dev/sdp') + volume_id = StringProperty(required=True) + past_volume_ids = ListProperty(item_type=str) + server = ReferenceProperty(Server, collection_name='volumes', + verbose_name='Server Attached To') + volume_state = CalculatedProperty(verbose_name="Volume State", + calculated_type=str, use_method=True) + attachment_state = CalculatedProperty(verbose_name="Attachment State", + calculated_type=str, use_method=True) + size = CalculatedProperty(verbose_name="Size (GB)", + calculated_type=int, use_method=True) + + @classmethod + def create(cls, **params): + getter = CommandLineGetter() + getter.get(cls, params) + region = params.get('region') + ec2 = region.connect() + zone = params.get('zone') + size = params.get('size') + ebs_volume = ec2.create_volume(size, zone.name) + v = cls() + v.ec2 = ec2 + v.volume_id = ebs_volume.id + v.name = params.get('name') + v.mount_point = params.get('mount_point') + v.device = params.get('device') + v.region_name = region.name + v.zone_name = zone.name + v.put() + return v + + @classmethod + def create_from_volume_id(cls, region_name, volume_id, name): + vol = None + ec2 = boto.ec2.connect_to_region(region_name) + rs = ec2.get_all_volumes([volume_id]) + if len(rs) == 1: + v = rs[0] + vol = cls() + vol.volume_id = v.id + vol.name = name + vol.region_name = v.region.name + vol.zone_name = v.zone + vol.put() + return vol + + def create_from_latest_snapshot(self, name, size=None): + snapshot = self.get_snapshots()[-1] + return self.create_from_snapshot(name, snapshot, size) + + def create_from_snapshot(self, name, snapshot, size=None): + if size < self.size: + size = self.size + ec2 = self.get_ec2_connection() + if self.zone_name == None or self.zone_name == '': + # deal with the migration case where the zone is not set in the logical volume: + current_volume = ec2.get_all_volumes([self.volume_id])[0] + self.zone_name = current_volume.zone + ebs_volume = ec2.create_volume(size, self.zone_name, snapshot) + v = Volume() + v.ec2 = self.ec2 + v.volume_id = ebs_volume.id + v.name = name + v.mount_point = self.mount_point + v.device = self.device + v.region_name = self.region_name + v.zone_name = self.zone_name + v.put() + return v + + def get_ec2_connection(self): + if self.server: + return self.server.ec2 + if not hasattr(self, 'ec2') or self.ec2 == None: + self.ec2 = boto.ec2.connect_to_region(self.region_name) + return self.ec2 + + def _volume_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].volume_state() + + def _attachment_state(self): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + return rs[0].attachment_state() + + def _size(self): + if not hasattr(self, '__size'): + ec2 = self.get_ec2_connection() + rs = ec2.get_all_volumes([self.volume_id]) + self.__size = rs[0].size + return self.__size + + def install_xfs(self): + if self.server: + self.server.install('xfsprogs xfsdump') + + def get_snapshots(self): + """ + Returns a list of all completed snapshots for this volume ID. + """ + ec2 = self.get_ec2_connection() + rs = ec2.get_all_snapshots() + all_vols = [self.volume_id] + self.past_volume_ids + snaps = [] + for snapshot in rs: + if snapshot.volume_id in all_vols: + if snapshot.progress == '100%': + snapshot.date = dateutil.parser.parse(snapshot.start_time) + snapshot.keep = True + snaps.append(snapshot) + snaps.sort(cmp=lambda x,y: cmp(x.date, y.date)) + return snaps + + def attach(self, server=None): + if self.attachment_state == 'attached': + print 'already attached' + return None + if server: + self.server = server + self.put() + ec2 = self.get_ec2_connection() + ec2.attach_volume(self.volume_id, self.server.instance_id, self.device) + + def detach(self, force=False): + state = self.attachment_state + if state == 'available' or state == None or state == 'detaching': + print 'already detached' + return None + ec2 = self.get_ec2_connection() + ec2.detach_volume(self.volume_id, self.server.instance_id, self.device, force) + self.server = None + self.put() + + def checkfs(self, use_cmd=None): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + # detemine state of file system on volume, only works if attached + if use_cmd: + cmd = use_cmd + else: + cmd = self.server.get_cmdshell() + status = cmd.run('xfs_check %s' % self.device) + if not use_cmd: + cmd.close() + if status[1].startswith('bad superblock magic number 0'): + return False + return True + + def wait(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + with closing(self.server.get_cmdshell()) as cmd: + # wait for the volume device to appear + cmd = self.server.get_cmdshell() + while not cmd.exists(self.device): + boto.log.info('%s still does not exist, waiting 10 seconds' % self.device) + time.sleep(10) + + def format(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + status = None + with closing(self.server.get_cmdshell()) as cmd: + if not self.checkfs(cmd): + boto.log.info('make_fs...') + status = cmd.run('mkfs -t xfs %s' % self.device) + return status + + def mount(self): + if self.server == None: + raise ValueError, 'server attribute must be set to run this command' + boto.log.info('handle_mount_point') + with closing(self.server.get_cmdshell()) as cmd: + cmd = self.server.get_cmdshell() + if not cmd.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + cmd.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + status = cmd.run('mount -l') + lines = status[1].split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + cmd.run('umount %s' % self.mount_point) + cmd.run('mount %s /tmp' % t[0]) + cmd.run('chmod 777 /tmp') + break + # Mount up our new EBS volume onto mount_point + cmd.run("mount %s %s" % (self.device, self.mount_point)) + cmd.run('xfs_growfs %s' % self.mount_point) + + def make_ready(self, server): + self.server = server + self.put() + self.install_xfs() + self.attach() + self.wait() + self.format() + self.mount() + + def freeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -f %s" % self.mount_point) + + def unfreeze(self): + if self.server: + return self.server.run("/usr/sbin/xfs_freeze -u %s" % self.mount_point) + + def snapshot(self): + # if this volume is attached to a server + # we need to freeze the XFS file system + try: + self.freeze() + if self.server == None: + snapshot = self.get_ec2_connection().create_snapshot(self.volume_id) + else: + snapshot = self.server.ec2.create_snapshot(self.volume_id) + boto.log.info('Snapshot of Volume %s created: %s' % (self.name, snapshot)) + except Exception: + boto.log.info('Snapshot error') + boto.log.info(traceback.format_exc()) + finally: + status = self.unfreeze() + return status + + def get_snapshot_range(self, snaps, start_date=None, end_date=None): + l = [] + for snap in snaps: + if start_date and end_date: + if snap.date >= start_date and snap.date <= end_date: + l.append(snap) + elif start_date: + if snap.date >= start_date: + l.append(snap) + elif end_date: + if snap.date <= end_date: + l.append(snap) + else: + l.append(snap) + return l + + def trim_snapshots(self, delete=False): + """ + Trim the number of snapshots for this volume. This method always + keeps the oldest snapshot. It then uses the parameters passed in + to determine how many others should be kept. + + The algorithm is to keep all snapshots from the current day. Then + it will keep the first snapshot of the day for the previous seven days. + Then, it will keep the first snapshot of the week for the previous + four weeks. After than, it will keep the first snapshot of the month + for as many months as there are. + + """ + snaps = self.get_snapshots() + # Always keep the oldest and the newest + if len(snaps) <= 2: + return snaps + snaps = snaps[1:-1] + now = datetime.datetime.now(snaps[0].date.tzinfo) + midnight = datetime.datetime(year=now.year, month=now.month, + day=now.day, tzinfo=now.tzinfo) + # Keep the first snapshot from each day of the previous week + one_week = datetime.timedelta(days=7, seconds=60*60) + print midnight-one_week, midnight + previous_week = self.get_snapshot_range(snaps, midnight-one_week, midnight) + print previous_week + if not previous_week: + return snaps + current_day = None + for snap in previous_week: + if current_day and current_day == snap.date.day: + snap.keep = False + else: + current_day = snap.date.day + # Get ourselves onto the next full week boundary + if previous_week: + week_boundary = previous_week[0].date + if week_boundary.weekday() != 0: + delta = datetime.timedelta(days=week_boundary.weekday()) + week_boundary = week_boundary - delta + # Keep one within this partial week + partial_week = self.get_snapshot_range(snaps, week_boundary, previous_week[0].date) + if len(partial_week) > 1: + for snap in partial_week[1:]: + snap.keep = False + # Keep the first snapshot of each week for the previous 4 weeks + for i in range(0,4): + weeks_worth = self.get_snapshot_range(snaps, week_boundary-one_week, week_boundary) + if len(weeks_worth) > 1: + for snap in weeks_worth[1:]: + snap.keep = False + week_boundary = week_boundary - one_week + # Now look through all remaining snaps and keep one per month + remainder = self.get_snapshot_range(snaps, end_date=week_boundary) + current_month = None + for snap in remainder: + if current_month and current_month == snap.date.month: + snap.keep = False + else: + current_month = snap.date.month + if delete: + for snap in snaps: + if not snap.keep: + boto.log.info('Deleting %s(%s) for %s' % (snap, snap.date, self.name)) + snap.delete() + return snaps + + def grow(self, size): + pass + + def copy(self, snapshot): + pass + + def get_snapshot_from_date(self, date): + pass + + def delete(self, delete_ebs_volume=False): + if delete_ebs_volume: + self.detach() + ec2 = self.get_ec2_connection() + ec2.delete_volume(self.volume_id) + Model.delete(self) + + def archive(self): + # snapshot volume, trim snaps, delete volume-id + pass + + diff --git a/backup/src/boto/mashups/__init__.py b/backup/src/boto/mashups/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/backup/src/boto/mashups/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/backup/src/boto/mashups/interactive.py b/backup/src/boto/mashups/interactive.py new file mode 100644 index 0000000..b80e661 --- /dev/null +++ b/backup/src/boto/mashups/interactive.py @@ -0,0 +1,97 @@ +# Copyright (C) 2003-2007 Robey Pointer +# +# This file is part of paramiko. +# +# Paramiko is free software; you can redistribute it and/or modify it under the +# terms of the GNU Lesser General Public License as published by the Free +# Software Foundation; either version 2.1 of the License, or (at your option) +# any later version. +# +# Paramiko is distrubuted in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more +# details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with Paramiko; if not, write to the Free Software Foundation, Inc., +# 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA. + + +import socket +import sys + +# windows does not have termios... +try: + import termios + import tty + has_termios = True +except ImportError: + has_termios = False + + +def interactive_shell(chan): + if has_termios: + posix_shell(chan) + else: + windows_shell(chan) + + +def posix_shell(chan): + import select + + oldtty = termios.tcgetattr(sys.stdin) + try: + tty.setraw(sys.stdin.fileno()) + tty.setcbreak(sys.stdin.fileno()) + chan.settimeout(0.0) + + while True: + r, w, e = select.select([chan, sys.stdin], [], []) + if chan in r: + try: + x = chan.recv(1024) + if len(x) == 0: + print '\r\n*** EOF\r\n', + break + sys.stdout.write(x) + sys.stdout.flush() + except socket.timeout: + pass + if sys.stdin in r: + x = sys.stdin.read(1) + if len(x) == 0: + break + chan.send(x) + + finally: + termios.tcsetattr(sys.stdin, termios.TCSADRAIN, oldtty) + + +# thanks to Mike Looijmans for this code +def windows_shell(chan): + import threading + + sys.stdout.write("Line-buffered terminal emulation. Press F6 or ^Z to send EOF.\r\n\r\n") + + def writeall(sock): + while True: + data = sock.recv(256) + if not data: + sys.stdout.write('\r\n*** EOF ***\r\n\r\n') + sys.stdout.flush() + break + sys.stdout.write(data) + sys.stdout.flush() + + writer = threading.Thread(target=writeall, args=(chan,)) + writer.start() + + try: + while True: + d = sys.stdin.read(1) + if not d: + break + chan.send(d) + except EOFError: + # user hit ^Z or F6 + pass diff --git a/backup/src/boto/mashups/iobject.py b/backup/src/boto/mashups/iobject.py new file mode 100644 index 0000000..a226b5c --- /dev/null +++ b/backup/src/boto/mashups/iobject.py @@ -0,0 +1,115 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os + +def int_val_fn(v): + try: + int(v) + return True + except: + return False + +class IObject(object): + + def choose_from_list(self, item_list, search_str='', + prompt='Enter Selection'): + if not item_list: + print 'No Choices Available' + return + choice = None + while not choice: + n = 1 + choices = [] + for item in item_list: + if isinstance(item, str): + print '[%d] %s' % (n, item) + choices.append(item) + n += 1 + else: + obj, id, desc = item + if desc: + if desc.find(search_str) >= 0: + print '[%d] %s - %s' % (n, id, desc) + choices.append(obj) + n += 1 + else: + if id.find(search_str) >= 0: + print '[%d] %s' % (n, id) + choices.append(obj) + n += 1 + if choices: + val = raw_input('%s[1-%d]: ' % (prompt, len(choices))) + if val.startswith('/'): + search_str = val[1:] + else: + try: + int_val = int(val) + if int_val == 0: + return None + choice = choices[int_val-1] + except ValueError: + print '%s is not a valid choice' % val + except IndexError: + print '%s is not within the range[1-%d]' % (val, + len(choices)) + else: + print "No objects matched your pattern" + search_str = '' + return choice + + def get_string(self, prompt, validation_fn=None): + okay = False + while not okay: + val = raw_input('%s: ' % prompt) + if validation_fn: + okay = validation_fn(val) + if not okay: + print 'Invalid value: %s' % val + else: + okay = True + return val + + def get_filename(self, prompt): + okay = False + val = '' + while not okay: + val = raw_input('%s: %s' % (prompt, val)) + val = os.path.expanduser(val) + if os.path.isfile(val): + okay = True + elif os.path.isdir(val): + path = val + val = self.choose_from_list(os.listdir(path)) + if val: + val = os.path.join(path, val) + okay = True + else: + val = '' + else: + print 'Invalid value: %s' % val + val = '' + return val + + def get_int(self, prompt): + s = self.get_string(prompt, int_val_fn) + return int(s) + diff --git a/backup/src/boto/mashups/order.py b/backup/src/boto/mashups/order.py new file mode 100644 index 0000000..6efdc3e --- /dev/null +++ b/backup/src/boto/mashups/order.py @@ -0,0 +1,211 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 order for servers +""" + +import boto +import boto.ec2 +from boto.mashups.server import Server, ServerSet +from boto.mashups.iobject import IObject +from boto.pyami.config import Config +from boto.sdb.persist import get_domain, set_domain +import time, StringIO + +InstanceTypes = ['m1.small', 'm1.large', 'm1.xlarge', 'c1.medium', 'c1.xlarge'] + +class Item(IObject): + + def __init__(self): + self.region = None + self.name = None + self.instance_type = None + self.quantity = 0 + self.zone = None + self.ami = None + self.groups = [] + self.key = None + self.ec2 = None + self.config = None + + def set_userdata(self, key, value): + self.userdata[key] = value + + def get_userdata(self, key): + return self.userdata[key] + + def set_region(self, region=None): + if region: + self.region = region + else: + l = [(r, r.name, r.endpoint) for r in boto.ec2.regions()] + self.region = self.choose_from_list(l, prompt='Choose Region') + + def set_name(self, name=None): + if name: + self.name = name + else: + self.name = self.get_string('Name') + + def set_instance_type(self, instance_type=None): + if instance_type: + self.instance_type = instance_type + else: + self.instance_type = self.choose_from_list(InstanceTypes, 'Instance Type') + + def set_quantity(self, n=0): + if n > 0: + self.quantity = n + else: + self.quantity = self.get_int('Quantity') + + def set_zone(self, zone=None): + if zone: + self.zone = zone + else: + l = [(z, z.name, z.state) for z in self.ec2.get_all_zones()] + self.zone = self.choose_from_list(l, prompt='Choose Availability Zone') + + def set_ami(self, ami=None): + if ami: + self.ami = ami + else: + l = [(a, a.id, a.location) for a in self.ec2.get_all_images()] + self.ami = self.choose_from_list(l, prompt='Choose AMI') + + def add_group(self, group=None): + if group: + self.groups.append(group) + else: + l = [(s, s.name, s.description) for s in self.ec2.get_all_security_groups()] + self.groups.append(self.choose_from_list(l, prompt='Choose Security Group')) + + def set_key(self, key=None): + if key: + self.key = key + else: + l = [(k, k.name, '') for k in self.ec2.get_all_key_pairs()] + self.key = self.choose_from_list(l, prompt='Choose Keypair') + + def update_config(self): + if not self.config.has_section('Credentials'): + self.config.add_section('Credentials') + self.config.set('Credentials', 'aws_access_key_id', self.ec2.aws_access_key_id) + self.config.set('Credentials', 'aws_secret_access_key', self.ec2.aws_secret_access_key) + if not self.config.has_section('Pyami'): + self.config.add_section('Pyami') + sdb_domain = get_domain() + if sdb_domain: + self.config.set('Pyami', 'server_sdb_domain', sdb_domain) + self.config.set('Pyami', 'server_sdb_name', self.name) + + def set_config(self, config_path=None): + if not config_path: + config_path = self.get_filename('Specify Config file') + self.config = Config(path=config_path) + + def get_userdata_string(self): + s = StringIO.StringIO() + self.config.write(s) + return s.getvalue() + + def enter(self, **params): + self.region = params.get('region', self.region) + if not self.region: + self.set_region() + self.ec2 = self.region.connect() + self.name = params.get('name', self.name) + if not self.name: + self.set_name() + self.instance_type = params.get('instance_type', self.instance_type) + if not self.instance_type: + self.set_instance_type() + self.zone = params.get('zone', self.zone) + if not self.zone: + self.set_zone() + self.quantity = params.get('quantity', self.quantity) + if not self.quantity: + self.set_quantity() + self.ami = params.get('ami', self.ami) + if not self.ami: + self.set_ami() + self.groups = params.get('groups', self.groups) + if not self.groups: + self.add_group() + self.key = params.get('key', self.key) + if not self.key: + self.set_key() + self.config = params.get('config', self.config) + if not self.config: + self.set_config() + self.update_config() + +class Order(IObject): + + def __init__(self): + self.items = [] + self.reservation = None + + def add_item(self, **params): + item = Item() + item.enter(**params) + self.items.append(item) + + def display(self): + print 'This Order consists of the following items' + print + print 'QTY\tNAME\tTYPE\nAMI\t\tGroups\t\t\tKeyPair' + for item in self.items: + print '%s\t%s\t%s\t%s\t%s\t%s' % (item.quantity, item.name, item.instance_type, + item.ami.id, item.groups, item.key.name) + + def place(self, block=True): + if get_domain() == None: + print 'SDB Persistence Domain not set' + domain_name = self.get_string('Specify SDB Domain') + set_domain(domain_name) + s = ServerSet() + for item in self.items: + r = item.ami.run(min_count=1, max_count=item.quantity, + key_name=item.key.name, user_data=item.get_userdata_string(), + security_groups=item.groups, instance_type=item.instance_type, + placement=item.zone.name) + if block: + states = [i.state for i in r.instances] + if states.count('running') != len(states): + print states + time.sleep(15) + states = [i.update() for i in r.instances] + for i in r.instances: + server = Server() + server.name = item.name + server.instance_id = i.id + server.reservation = r + server.save() + s.append(server) + if len(s) == 1: + return s[0] + else: + return s + + + diff --git a/backup/src/boto/mashups/server.py b/backup/src/boto/mashups/server.py new file mode 100644 index 0000000..6cea106 --- /dev/null +++ b/backup/src/boto/mashups/server.py @@ -0,0 +1,395 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +High-level abstraction of an EC2 server +""" +import boto +import boto.utils +from boto.mashups.iobject import IObject +from boto.pyami.config import Config, BotoConfigPath +from boto.mashups.interactive import interactive_shell +from boto.sdb.db.model import Model +from boto.sdb.db.property import StringProperty +import os +import StringIO + + +class ServerSet(list): + + def __getattr__(self, name): + results = [] + is_callable = False + for server in self: + try: + val = getattr(server, name) + if callable(val): + is_callable = True + results.append(val) + except: + results.append(None) + if is_callable: + self.map_list = results + return self.map + return results + + def map(self, *args): + results = [] + for fn in self.map_list: + results.append(fn(*args)) + return results + +class Server(Model): + + @property + def ec2(self): + if self._ec2 is None: + self._ec2 = boto.connect_ec2() + return self._ec2 + + @classmethod + def Inventory(cls): + """ + Returns a list of Server instances, one for each Server object + persisted in the db + """ + l = ServerSet() + rs = cls.find() + for server in rs: + l.append(server) + return l + + @classmethod + def Register(cls, name, instance_id, description=''): + s = cls() + s.name = name + s.instance_id = instance_id + s.description = description + s.save() + return s + + def __init__(self, id=None, **kw): + Model.__init__(self, id, **kw) + self._reservation = None + self._instance = None + self._ssh_client = None + self._pkey = None + self._config = None + self._ec2 = None + + name = StringProperty(unique=True, verbose_name="Name") + instance_id = StringProperty(verbose_name="Instance ID") + config_uri = StringProperty() + ami_id = StringProperty(verbose_name="AMI ID") + zone = StringProperty(verbose_name="Availability Zone") + security_group = StringProperty(verbose_name="Security Group", default="default") + key_name = StringProperty(verbose_name="Key Name") + elastic_ip = StringProperty(verbose_name="Elastic IP") + instance_type = StringProperty(verbose_name="Instance Type") + description = StringProperty(verbose_name="Description") + log = StringProperty() + + def setReadOnly(self, value): + raise AttributeError + + def getInstance(self): + if not self._instance: + if self.instance_id: + try: + rs = self.ec2.get_all_instances([self.instance_id]) + except: + return None + if len(rs) > 0: + self._reservation = rs[0] + self._instance = self._reservation.instances[0] + return self._instance + + instance = property(getInstance, setReadOnly, None, 'The Instance for the server') + + def getAMI(self): + if self.instance: + return self.instance.image_id + + ami = property(getAMI, setReadOnly, None, 'The AMI for the server') + + def getStatus(self): + if self.instance: + self.instance.update() + return self.instance.state + + status = property(getStatus, setReadOnly, None, + 'The status of the server') + + def getHostname(self): + if self.instance: + return self.instance.public_dns_name + + hostname = property(getHostname, setReadOnly, None, + 'The public DNS name of the server') + + def getPrivateHostname(self): + if self.instance: + return self.instance.private_dns_name + + private_hostname = property(getPrivateHostname, setReadOnly, None, + 'The private DNS name of the server') + + def getLaunchTime(self): + if self.instance: + return self.instance.launch_time + + launch_time = property(getLaunchTime, setReadOnly, None, + 'The time the Server was started') + + def getConsoleOutput(self): + if self.instance: + return self.instance.get_console_output() + + console_output = property(getConsoleOutput, setReadOnly, None, + 'Retrieve the console output for server') + + def getGroups(self): + if self._reservation: + return self._reservation.groups + else: + return None + + groups = property(getGroups, setReadOnly, None, + 'The Security Groups controlling access to this server') + + def getConfig(self): + if not self._config: + remote_file = BotoConfigPath + local_file = '%s.ini' % self.instance.id + self.get_file(remote_file, local_file) + self._config = Config(local_file) + return self._config + + def setConfig(self, config): + local_file = '%s.ini' % self.instance.id + fp = open(local_file) + config.write(fp) + fp.close() + self.put_file(local_file, BotoConfigPath) + self._config = config + + config = property(getConfig, setConfig, None, + 'The instance data for this server') + + def set_config(self, config): + """ + Set SDB based config + """ + self._config = config + self._config.dump_to_sdb("botoConfigs", self.id) + + def load_config(self): + self._config = Config(do_load=False) + self._config.load_from_sdb("botoConfigs", self.id) + + def stop(self): + if self.instance: + self.instance.stop() + + def start(self): + self.stop() + ec2 = boto.connect_ec2() + ami = ec2.get_all_images(image_ids = [str(self.ami_id)])[0] + groups = ec2.get_all_security_groups(groupnames=[str(self.security_group)]) + if not self._config: + self.load_config() + if not self._config.has_section("Credentials"): + self._config.add_section("Credentials") + self._config.set("Credentials", "aws_access_key_id", ec2.aws_access_key_id) + self._config.set("Credentials", "aws_secret_access_key", ec2.aws_secret_access_key) + + if not self._config.has_section("Pyami"): + self._config.add_section("Pyami") + + if self._manager.domain: + self._config.set('Pyami', 'server_sdb_domain', self._manager.domain.name) + self._config.set("Pyami", 'server_sdb_name', self.name) + + cfg = StringIO.StringIO() + self._config.write(cfg) + cfg = cfg.getvalue() + r = ami.run(min_count=1, + max_count=1, + key_name=self.key_name, + security_groups = groups, + instance_type = self.instance_type, + placement = self.zone, + user_data = cfg) + i = r.instances[0] + self.instance_id = i.id + self.put() + if self.elastic_ip: + ec2.associate_address(self.instance_id, self.elastic_ip) + + def reboot(self): + if self.instance: + self.instance.reboot() + + def get_ssh_client(self, key_file=None, host_key_file='~/.ssh/known_hosts', + uname='root'): + import paramiko + if not self.instance: + print 'No instance yet!' + return + if not self._ssh_client: + if not key_file: + iobject = IObject() + key_file = iobject.get_filename('Path to OpenSSH Key file') + self._pkey = paramiko.RSAKey.from_private_key_file(key_file) + self._ssh_client = paramiko.SSHClient() + self._ssh_client.load_system_host_keys() + self._ssh_client.load_host_keys(os.path.expanduser(host_key_file)) + self._ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self._ssh_client.connect(self.instance.public_dns_name, + username=uname, pkey=self._pkey) + return self._ssh_client + + def get_file(self, remotepath, localpath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.get(remotepath, localpath) + + def put_file(self, localpath, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + sftp_client.put(localpath, remotepath) + + def listdir(self, remotepath): + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + return sftp_client.listdir(remotepath) + + def shell(self, key_file=None): + ssh_client = self.get_ssh_client(key_file) + channel = ssh_client.invoke_shell() + interactive_shell(channel) + + def bundle_image(self, prefix, key_file, cert_file, size): + print 'bundling image...' + print '\tcopying cert and pk over to /mnt directory on server' + ssh_client = self.get_ssh_client() + sftp_client = ssh_client.open_sftp() + path, name = os.path.split(key_file) + remote_key_file = '/mnt/%s' % name + self.put_file(key_file, remote_key_file) + path, name = os.path.split(cert_file) + remote_cert_file = '/mnt/%s' % name + self.put_file(cert_file, remote_cert_file) + print '\tdeleting %s' % BotoConfigPath + # delete the metadata.ini file if it exists + try: + sftp_client.remove(BotoConfigPath) + except: + pass + command = 'sudo ec2-bundle-vol ' + command += '-c %s -k %s ' % (remote_cert_file, remote_key_file) + command += '-u %s ' % self._reservation.owner_id + command += '-p %s ' % prefix + command += '-s %d ' % size + command += '-d /mnt ' + if self.instance.instance_type == 'm1.small' or self.instance_type == 'c1.medium': + command += '-r i386' + else: + command += '-r x86_64' + print '\t%s' % command + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def upload_bundle(self, bucket, prefix): + print 'uploading bundle...' + command = 'ec2-upload-bundle ' + command += '-m /mnt/%s.manifest.xml ' % prefix + command += '-b %s ' % bucket + command += '-a %s ' % self.ec2.aws_access_key_id + command += '-s %s ' % self.ec2.aws_secret_access_key + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' + + def create_image(self, bucket=None, prefix=None, key_file=None, cert_file=None, size=None): + iobject = IObject() + if not bucket: + bucket = iobject.get_string('Name of S3 bucket') + if not prefix: + prefix = iobject.get_string('Prefix for AMI file') + if not key_file: + key_file = iobject.get_filename('Path to RSA private key file') + if not cert_file: + cert_file = iobject.get_filename('Path to RSA public cert file') + if not size: + size = iobject.get_int('Size (in MB) of bundled image') + self.bundle_image(prefix, key_file, cert_file, size) + self.upload_bundle(bucket, prefix) + print 'registering image...' + self.image_id = self.ec2.register_image('%s/%s.manifest.xml' % (bucket, prefix)) + return self.image_id + + def attach_volume(self, volume, device="/dev/sdp"): + """ + Attach an EBS volume to this server + + :param volume: EBS Volume to attach + :type volume: boto.ec2.volume.Volume + + :param device: Device to attach to (default to /dev/sdp) + :type device: string + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.attach_volume(volume_id=volume_id, instance_id=self.instance_id, device=device) + + def detach_volume(self, volume): + """ + Detach an EBS volume from this server + + :param volume: EBS Volume to detach + :type volume: boto.ec2.volume.Volume + """ + if hasattr(volume, "id"): + volume_id = volume.id + else: + volume_id = volume + return self.ec2.detach_volume(volume_id=volume_id, instance_id=self.instance_id) + + def install_package(self, package_name): + print 'installing %s...' % package_name + command = 'yum -y install %s' % package_name + print '\t%s' % command + ssh_client = self.get_ssh_client() + t = ssh_client.exec_command(command) + response = t[1].read() + print '\t%s' % response + print '\t%s' % t[2].read() + print '...complete!' diff --git a/backup/src/boto/mturk/__init__.py b/backup/src/boto/mturk/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/backup/src/boto/mturk/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/backup/src/boto/mturk/connection.py b/backup/src/boto/mturk/connection.py new file mode 100644 index 0000000..619697f --- /dev/null +++ b/backup/src/boto/mturk/connection.py @@ -0,0 +1,887 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import datetime +import itertools + +from boto import handler +from boto import config +from boto.mturk.price import Price +import boto.mturk.notification +from boto.connection import AWSQueryConnection +from boto.exception import EC2ResponseError +from boto.resultset import ResultSet +from boto.mturk.question import QuestionForm, ExternalQuestion + +class MTurkRequestError(EC2ResponseError): + "Error for MTurk Requests" + # todo: subclass from an abstract parent of EC2ResponseError + +class MTurkConnection(AWSQueryConnection): + + APIVersion = '2008-08-02' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=False, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host=None, debug=0, + https_connection_factory=None): + if not host: + if config.has_option('MTurk', 'sandbox') and config.get('MTurk', 'sandbox') == 'True': + host = 'mechanicalturk.sandbox.amazonaws.com' + else: + host = 'mechanicalturk.amazonaws.com' + + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, proxy_port, + proxy_user, proxy_pass, host, debug, + https_connection_factory) + + def _required_auth_capability(self): + return ['mturk'] + + def get_account_balance(self): + """ + """ + params = {} + return self._process_request('GetAccountBalance', params, + [('AvailableBalance', Price), + ('OnHoldBalance', Price)]) + + def register_hit_type(self, title, description, reward, duration, + keywords=None, approval_delay=None, qual_req=None): + """ + Register a new HIT Type + title, description are strings + reward is a Price object + duration can be a timedelta, or an object castable to an int + """ + params = dict( + Title=title, + Description=description, + AssignmentDurationInSeconds= + self.duration_as_seconds(duration), + ) + params.update(MTurkConnection.get_price_as_price(reward).get_as_params('Reward')) + + if keywords: + params['Keywords'] = self.get_keywords_as_string(keywords) + + if approval_delay is not None: + d = self.duration_as_seconds(approval_delay) + params['AutoApprovalDelayInSeconds'] = d + + if qual_req is not None: + params.update(qual_req.get_as_params()) + + return self._process_request('RegisterHITType', params) + + def set_email_notification(self, hit_type, email, event_types=None): + """ + Performs a SetHITTypeNotification operation to set email + notification for a specified HIT type + """ + return self._set_notification(hit_type, 'Email', email, event_types) + + def set_rest_notification(self, hit_type, url, event_types=None): + """ + Performs a SetHITTypeNotification operation to set REST notification + for a specified HIT type + """ + return self._set_notification(hit_type, 'REST', url, event_types) + + def _set_notification(self, hit_type, transport, destination, event_types=None): + """ + Common SetHITTypeNotification operation to set notification for a + specified HIT type + """ + assert type(hit_type) is str, "hit_type argument should be a string." + + params = {'HITTypeId': hit_type} + + # from the Developer Guide: + # The 'Active' parameter is optional. If omitted, the active status of + # the HIT type's notification specification is unchanged. All HIT types + # begin with their notification specifications in the "inactive" status. + notification_params = {'Destination': destination, + 'Transport': transport, + 'Version': boto.mturk.notification.NotificationMessage.NOTIFICATION_VERSION, + 'Active': True, + } + + # add specific event types if required + if event_types: + self.build_list_params(notification_params, event_types, 'EventType') + + # Set up dict of 'Notification.1.Transport' etc. values + notification_rest_params = {} + num = 1 + for key in notification_params: + notification_rest_params['Notification.%d.%s' % (num, key)] = notification_params[key] + + # Update main params dict + params.update(notification_rest_params) + + # Execute operation + return self._process_request('SetHITTypeNotification', params) + + def create_hit(self, hit_type=None, question=None, + lifetime=datetime.timedelta(days=7), + max_assignments=1, + title=None, description=None, keywords=None, + reward=None, duration=datetime.timedelta(days=7), + approval_delay=None, annotation=None, + questions=None, qualifications=None, + response_groups=None): + """ + Creates a new HIT. + Returns a ResultSet + See: http://docs.amazonwebservices.com/AWSMechanicalTurkRequester/2006-10-31/ApiReference_CreateHITOperation.html + """ + + # handle single or multiple questions + neither = question is None and questions is None + both = question is not None and questions is not None + if neither or both: + raise ValueError("Must specify either question (single Question instance) or questions (list or QuestionForm instance), but not both") + + if question: + questions = [question] + question_param = QuestionForm(questions) + if isinstance(question, QuestionForm): + question_param = question + elif isinstance(question, ExternalQuestion): + question_param = question + + # Handle basic required arguments and set up params dict + params = {'Question': question_param.get_as_xml(), + 'LifetimeInSeconds' : + self.duration_as_seconds(lifetime), + 'MaxAssignments' : max_assignments, + } + + # if hit type specified then add it + # else add the additional required parameters + if hit_type: + params['HITTypeId'] = hit_type + else: + # Handle keywords + final_keywords = MTurkConnection.get_keywords_as_string(keywords) + + # Handle price argument + final_price = MTurkConnection.get_price_as_price(reward) + + final_duration = self.duration_as_seconds(duration) + + additional_params = dict( + Title=title, + Description=description, + Keywords=final_keywords, + AssignmentDurationInSeconds=final_duration, + ) + additional_params.update(final_price.get_as_params('Reward')) + + if approval_delay is not None: + d = self.duration_as_seconds(approval_delay) + additional_params['AutoApprovalDelayInSeconds'] = d + + # add these params to the others + params.update(additional_params) + + # add the annotation if specified + if annotation is not None: + params['RequesterAnnotation'] = annotation + + # Add the Qualifications if specified + if qualifications is not None: + params.update(qualifications.get_as_params()) + + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + # Submit + return self._process_request('CreateHIT', params, [('HIT', HIT),]) + + def change_hit_type_of_hit(self, hit_id, hit_type): + """ + Change the HIT type of an existing HIT. Note that the reward associated + with the new HIT type must match the reward of the current HIT type in + order for the operation to be valid. + \thit_id is a string + \thit_type is a string + """ + params = {'HITId' : hit_id, + 'HITTypeId': hit_type} + + return self._process_request('ChangeHITTypeOfHIT', params) + + def get_reviewable_hits(self, hit_type=None, status='Reviewable', + sort_by='Expiration', sort_direction='Ascending', + page_size=10, page_number=1): + """ + Retrieve the HITs that have a status of Reviewable, or HITs that + have a status of Reviewing, and that belong to the Requester + calling the operation. + """ + params = {'Status' : status, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + # Handle optional hit_type argument + if hit_type is not None: + params.update({'HITTypeId': hit_type}) + + return self._process_request('GetReviewableHITs', params, [('HIT', HIT),]) + + @staticmethod + def _get_pages(page_size, total_records): + """ + Given a page size (records per page) and a total number of + records, return the page numbers to be retrieved. + """ + pages = total_records/page_size+bool(total_records%page_size) + return range(1, pages+1) + + + def get_all_hits(self): + """ + Return all of a Requester's HITs + + Despite what search_hits says, it does not return all hits, but + instead returns a page of hits. This method will pull the hits + from the server 100 at a time, but will yield the results + iteratively, so subsequent requests are made on demand. + """ + page_size = 100 + search_rs = self.search_hits(page_size=page_size) + total_records = int(search_rs.TotalNumResults) + get_page_hits = lambda(page): self.search_hits(page_size=page_size, page_number=page) + page_nums = self._get_pages(page_size, total_records) + hit_sets = itertools.imap(get_page_hits, page_nums) + return itertools.chain.from_iterable(hit_sets) + + def search_hits(self, sort_by='CreationTime', sort_direction='Ascending', + page_size=10, page_number=1, response_groups=None): + """ + Return a page of a Requester's HITs, on behalf of the Requester. + The operation returns HITs of any status, except for HITs that + have been disposed with the DisposeHIT operation. + Note: + The SearchHITs operation does not accept any search parameters + that filter the results. + """ + params = {'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + + return self._process_request('SearchHITs', params, [('HIT', HIT),]) + + def get_assignments(self, hit_id, status=None, + sort_by='SubmitTime', sort_direction='Ascending', + page_size=10, page_number=1, response_groups=None): + """ + Retrieves completed assignments for a HIT. + Use this operation to retrieve the results for a HIT. + + The returned ResultSet will have the following attributes: + + NumResults + The number of assignments on the page in the filtered results + list, equivalent to the number of assignments being returned + by this call. + A non-negative integer + PageNumber + The number of the page in the filtered results list being + returned. + A positive integer + TotalNumResults + The total number of HITs in the filtered results list based + on this call. + A non-negative integer + + The ResultSet will contain zero or more Assignment objects + + """ + params = {'HITId' : hit_id, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + + if status is not None: + params['AssignmentStatus'] = status + + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + return self._process_request('GetAssignmentsForHIT', params, + [('Assignment', Assignment),]) + + def approve_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('ApproveAssignment', params) + + def reject_assignment(self, assignment_id, feedback=None): + """ + """ + params = {'AssignmentId' : assignment_id,} + if feedback: + params['RequesterFeedback'] = feedback + return self._process_request('RejectAssignment', params) + + def get_hit(self, hit_id, response_groups=None): + """ + """ + params = {'HITId' : hit_id,} + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + return self._process_request('GetHIT', params, [('HIT', HIT),]) + + def set_reviewing(self, hit_id, revert=None): + """ + Update a HIT with a status of Reviewable to have a status of Reviewing, + or reverts a Reviewing HIT back to the Reviewable status. + + Only HITs with a status of Reviewable can be updated with a status of + Reviewing. Similarly, only Reviewing HITs can be reverted back to a + status of Reviewable. + """ + params = {'HITId' : hit_id,} + if revert: + params['Revert'] = revert + return self._process_request('SetHITAsReviewing', params) + + def disable_hit(self, hit_id, response_groups=None): + """ + Remove a HIT from the Mechanical Turk marketplace, approves all + submitted assignments that have not already been approved or rejected, + and disposes of the HIT and all assignment data. + + Assignments for the HIT that have already been submitted, but not yet + approved or rejected, will be automatically approved. Assignments in + progress at the time of the call to DisableHIT will be approved once + the assignments are submitted. You will be charged for approval of + these assignments. DisableHIT completely disposes of the HIT and + all submitted assignment data. Assignment results data cannot be + retrieved for a HIT that has been disposed. + + It is not possible to re-enable a HIT once it has been disabled. + To make the work from a disabled HIT available again, create a new HIT. + """ + params = {'HITId' : hit_id,} + # Handle optional response groups argument + if response_groups: + self.build_list_params(params, response_groups, 'ResponseGroup') + + return self._process_request('DisableHIT', params) + + def dispose_hit(self, hit_id): + """ + Dispose of a HIT that is no longer needed. + + Only HITs in the "reviewable" state, with all submitted + assignments approved or rejected, can be disposed. A Requester + can call GetReviewableHITs to determine which HITs are + reviewable, then call GetAssignmentsForHIT to retrieve the + assignments. Disposing of a HIT removes the HIT from the + results of a call to GetReviewableHITs. """ + params = {'HITId' : hit_id,} + return self._process_request('DisposeHIT', params) + + def expire_hit(self, hit_id): + + """ + Expire a HIT that is no longer needed. + + The effect is identical to the HIT expiring on its own. The + HIT no longer appears on the Mechanical Turk web site, and no + new Workers are allowed to accept the HIT. Workers who have + accepted the HIT prior to expiration are allowed to complete + it or return it, or allow the assignment duration to elapse + (abandon the HIT). Once all remaining assignments have been + submitted, the expired HIT becomes"reviewable", and will be + returned by a call to GetReviewableHITs. + """ + params = {'HITId' : hit_id,} + return self._process_request('ForceExpireHIT', params) + + def extend_hit(self, hit_id, assignments_increment=None, expiration_increment=None): + """ + Increase the maximum number of assignments, or extend the + expiration date, of an existing HIT. + + NOTE: If a HIT has a status of Reviewable and the HIT is + extended to make it Available, the HIT will not be returned by + GetReviewableHITs, and its submitted assignments will not be + returned by GetAssignmentsForHIT, until the HIT is Reviewable + again. Assignment auto-approval will still happen on its + original schedule, even if the HIT has been extended. Be sure + to retrieve and approve (or reject) submitted assignments + before extending the HIT, if so desired. + """ + # must provide assignment *or* expiration increment + if (assignments_increment is None and expiration_increment is None) or \ + (assignments_increment is not None and expiration_increment is not None): + raise ValueError("Must specify either assignments_increment or expiration_increment, but not both") + + params = {'HITId' : hit_id,} + if assignments_increment: + params['MaxAssignmentsIncrement'] = assignments_increment + if expiration_increment: + params['ExpirationIncrementInSeconds'] = expiration_increment + + return self._process_request('ExtendHIT', params) + + def get_help(self, about, help_type='Operation'): + """ + Return information about the Mechanical Turk Service + operations and response group NOTE - this is basically useless + as it just returns the URL of the documentation + + help_type: either 'Operation' or 'ResponseGroup' + """ + params = {'About': about, 'HelpType': help_type,} + return self._process_request('Help', params) + + def grant_bonus(self, worker_id, assignment_id, bonus_price, reason): + """ + Issues a payment of money from your account to a Worker. To + be eligible for a bonus, the Worker must have submitted + results for one of your HITs, and have had those results + approved or rejected. This payment happens separately from the + reward you pay to the Worker when you approve the Worker's + assignment. The Bonus must be passed in as an instance of the + Price object. + """ + params = bonus_price.get_as_params('BonusAmount', 1) + params['WorkerId'] = worker_id + params['AssignmentId'] = assignment_id + params['Reason'] = reason + + return self._process_request('GrantBonus', params) + + def block_worker(self, worker_id, reason): + """ + Block a worker from working on my tasks. + """ + params = {'WorkerId': worker_id, 'Reason': reason} + + return self._process_request('BlockWorker', params) + + def unblock_worker(self, worker_id, reason): + """ + Unblock a worker from working on my tasks. + """ + params = {'WorkerId': worker_id, 'Reason': reason} + + return self._process_request('UnblockWorker', params) + + def notify_workers(self, worker_ids, subject, message_text): + """ + Send a text message to workers. + """ + params = {'WorkerId' : worker_ids, + 'Subject' : subject, + 'MessageText': message_text} + + return self._process_request('NotifyWorkers', params) + + def create_qualification_type(self, + name, + description, + status, + keywords=None, + retry_delay=None, + test=None, + answer_key=None, + answer_key_xml=None, + test_duration=None, + auto_granted=False, + auto_granted_value=1): + """ + Create a new Qualification Type. + + name: This will be visible to workers and must be unique for a + given requester. + + description: description shown to workers. Max 2000 characters. + + status: 'Active' or 'Inactive' + + keywords: list of keyword strings or comma separated string. + Max length of 1000 characters when concatenated with commas. + + retry_delay: number of seconds after requesting a + qualification the worker must wait before they can ask again. + If not specified, workers can only request this qualification + once. + + test: a QuestionForm + + answer_key: an XML string of your answer key, for automatically + scored qualification tests. + (Consider implementing an AnswerKey class for this to support.) + + test_duration: the number of seconds a worker has to complete the test. + + auto_granted: if True, requests for the Qualification are granted immediately. + Can't coexist with a test. + + auto_granted_value: auto_granted qualifications are given this value. + + """ + + params = {'Name' : name, + 'Description' : description, + 'QualificationTypeStatus' : status, + } + if retry_delay is not None: + params['RetryDelay'] = retry_delay + + if test is not None: + assert(isinstance(test, QuestionForm)) + assert(test_duration is not None) + params['Test'] = test.get_as_xml() + + if test_duration is not None: + params['TestDuration'] = test_duration + + if answer_key is not None: + if isinstance(answer_key, basestring): + params['AnswerKey'] = answer_key # xml + else: + raise TypeError + # Eventually someone will write an AnswerKey class. + + if auto_granted: + assert(test is False) + params['AutoGranted'] = True + params['AutoGrantedValue'] = auto_granted_value + + if keywords: + params['Keywords'] = self.get_keywords_as_string(keywords) + + return self._process_request('CreateQualificationType', params, + [('QualificationType', QualificationType),]) + + def get_qualification_type(self, qualification_type_id): + params = {'QualificationTypeId' : qualification_type_id } + return self._process_request('GetQualificationType', params, + [('QualificationType', QualificationType),]) + + def get_qualifications_for_qualification_type(self, qualification_type_id): + params = {'QualificationTypeId' : qualification_type_id } + return self._process_request('GetQualificationsForQualificationType', params, + [('QualificationType', QualificationType),]) + + def update_qualification_type(self, qualification_type_id, + description=None, + status=None, + retry_delay=None, + test=None, + answer_key=None, + test_duration=None, + auto_granted=None, + auto_granted_value=None): + + params = {'QualificationTypeId' : qualification_type_id } + + if description is not None: + params['Description'] = description + + if status is not None: + params['QualificationTypeStatus'] = status + + if retry_delay is not None: + params['RetryDelay'] = retry_delay + + if test is not None: + assert(isinstance(test, QuestionForm)) + params['Test'] = test.get_as_xml() + + if test_duration is not None: + params['TestDuration'] = test_duration + + if answer_key is not None: + if isinstance(answer_key, basestring): + params['AnswerKey'] = answer_key # xml + else: + raise TypeError + # Eventually someone will write an AnswerKey class. + + if auto_granted is not None: + params['AutoGranted'] = auto_granted + + if auto_granted_value is not None: + params['AutoGrantedValue'] = auto_granted_value + + return self._process_request('UpdateQualificationType', params, + [('QualificationType', QualificationType),]) + + def dispose_qualification_type(self, qualification_type_id): + """TODO: Document.""" + params = {'QualificationTypeId' : qualification_type_id} + return self._process_request('DisposeQualificationType', params) + + def search_qualification_types(self, query=None, sort_by='Name', + sort_direction='Ascending', page_size=10, + page_number=1, must_be_requestable=True, + must_be_owned_by_caller=True): + """TODO: Document.""" + params = {'Query' : query, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number, + 'MustBeRequestable' : must_be_requestable, + 'MustBeOwnedByCaller' : must_be_owned_by_caller} + return self._process_request('SearchQualificationTypes', params, + [('QualificationType', QualificationType),]) + + def get_qualification_requests(self, qualification_type_id, + sort_by='Expiration', + sort_direction='Ascending', page_size=10, + page_number=1): + """TODO: Document.""" + params = {'QualificationTypeId' : qualification_type_id, + 'SortProperty' : sort_by, + 'SortDirection' : sort_direction, + 'PageSize' : page_size, + 'PageNumber' : page_number} + return self._process_request('GetQualificationRequests', params, + [('QualificationRequest', QualificationRequest),]) + + def grant_qualification(self, qualification_request_id, integer_value=1): + """TODO: Document.""" + params = {'QualificationRequestId' : qualification_request_id, + 'IntegerValue' : integer_value} + return self._process_request('GrantQualification', params) + + def revoke_qualification(self, subject_id, qualification_type_id, + reason=None): + """TODO: Document.""" + params = {'SubjectId' : subject_id, + 'QualificationTypeId' : qualification_type_id, + 'Reason' : reason} + return self._process_request('RevokeQualification', params) + + def assign_qualification(self, qualification_type_id, worker_id, + value=1, send_notification=True): + params = {'QualificationTypeId' : qualification_type_id, + 'WorkerId' : worker_id, + 'IntegerValue' : value, + 'SendNotification' : send_notification, } + return self._process_request('AssignQualification', params) + + def _process_request(self, request_type, params, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + response = self.make_request(request_type, params, verb='POST') + return self._process_response(response, marker_elems) + + def _process_response(self, response, marker_elems=None): + """ + Helper to process the xml response from AWS + """ + body = response.read() + #print body + if '' not in body: + rs = ResultSet(marker_elems) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise MTurkRequestError(response.status, response.reason, body) + + @staticmethod + def get_keywords_as_string(keywords): + """ + Returns a comma+space-separated string of keywords from either + a list or a string + """ + if type(keywords) is list: + keywords = ', '.join(keywords) + if type(keywords) is str: + final_keywords = keywords + elif type(keywords) is unicode: + final_keywords = keywords.encode('utf-8') + elif keywords is None: + final_keywords = "" + else: + raise TypeError("keywords argument must be a string or a list of strings; got a %s" % type(keywords)) + return final_keywords + + @staticmethod + def get_price_as_price(reward): + """ + Returns a Price data structure from either a float or a Price + """ + if isinstance(reward, Price): + final_price = reward + else: + final_price = Price(reward) + return final_price + + @staticmethod + def duration_as_seconds(duration): + if isinstance(duration, datetime.timedelta): + duration = duration.days*86400 + duration.seconds + try: + duration = int(duration) + except TypeError: + raise TypeError("Duration must be a timedelta or int-castable, got %s" % type(duration)) + return duration + +class BaseAutoResultElement: + """ + Base class to automatically add attributes when parsing XML + """ + def __init__(self, connection): + pass + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + setattr(self, name, value) + +class HIT(BaseAutoResultElement): + """ + Class to extract a HIT structure from a response (used in ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. HITId, HITTypeId, CreationTime + """ + + # property helper to determine if HIT has expired + def _has_expired(self): + """ Has this HIT expired yet? """ + expired = False + if hasattr(self, 'Expiration'): + now = datetime.datetime.utcnow() + expiration = datetime.datetime.strptime(self.Expiration, '%Y-%m-%dT%H:%M:%SZ') + expired = (now >= expiration) + else: + raise ValueError("ERROR: Request for expired property, but no Expiration in HIT!") + return expired + + # are we there yet? + expired = property(_has_expired) + +class QualificationType(BaseAutoResultElement): + """ + Class to extract an QualificationType structure from a response (used in + ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. QualificationTypeId, CreationTime, Name, etc + """ + + pass + +class QualificationRequest(BaseAutoResultElement): + """ + Class to extract an QualificationRequest structure from a response (used in + ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. QualificationRequestId, QualificationTypeId, SubjectId, etc + + TODO: Ensure that Test and Answer attribute are treated properly if the + qualification requires a test. These attributes are XML-encoded. + """ + + pass + +class Assignment(BaseAutoResultElement): + """ + Class to extract an Assignment structure from a response (used in + ResultSet) + + Will have attributes named as per the Developer Guide, + e.g. AssignmentId, WorkerId, HITId, Answer, etc + """ + + def __init__(self, connection): + BaseAutoResultElement.__init__(self, connection) + self.answers = [] + + def endElement(self, name, value, connection): + # the answer consists of embedded XML, so it needs to be parsed independantly + if name == 'Answer': + answer_rs = ResultSet([('Answer', QuestionFormAnswer),]) + h = handler.XmlHandler(answer_rs, connection) + value = connection.get_utf8_value(value) + xml.sax.parseString(value, h) + self.answers.append(answer_rs) + else: + BaseAutoResultElement.endElement(self, name, value, connection) + +class QuestionFormAnswer(BaseAutoResultElement): + """ + Class to extract Answers from inside the embedded XML + QuestionFormAnswers element inside the Answer element which is + part of the Assignment structure + + A QuestionFormAnswers element contains an Answer element for each + question in the HIT or Qualification test for which the Worker + provided an answer. Each Answer contains a QuestionIdentifier + element whose value corresponds to the QuestionIdentifier of a + Question in the QuestionForm. See the QuestionForm data structure + for more information about questions and answer specifications. + + If the question expects a free-text answer, the Answer element + contains a FreeText element. This element contains the Worker's + answer + + *NOTE* - currently really only supports free-text and selection answers + """ + + def __init__(self, connection): + BaseAutoResultElement.__init__(self, connection) + self.fields = [] + self.qid = None + + def endElement(self, name, value, connection): + if name == 'QuestionIdentifier': + self.qid = value + elif name in ['FreeText', 'SelectionIdentifier'] and self.qid: + self.fields.append((self.qid,value)) + elif name == 'Answer': + self.qid = None diff --git a/backup/src/boto/mturk/notification.py b/backup/src/boto/mturk/notification.py new file mode 100644 index 0000000..2aa99ca --- /dev/null +++ b/backup/src/boto/mturk/notification.py @@ -0,0 +1,94 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Provides NotificationMessage and Event classes, with utility methods, for +implementations of the Mechanical Turk Notification API. +""" + +import hmac +try: + from hashlib import sha1 as sha +except ImportError: + import sha +import base64 +import re + +class NotificationMessage: + + NOTIFICATION_WSDL = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurk/2006-05-05/AWSMechanicalTurkRequesterNotification.wsdl" + NOTIFICATION_VERSION = '2006-05-05' + + SERVICE_NAME = "AWSMechanicalTurkRequesterNotification" + OPERATION_NAME = "Notify" + + EVENT_PATTERN = r"Event\.(?P\d+)\.(?P\w+)" + EVENT_RE = re.compile(EVENT_PATTERN) + + def __init__(self, d): + """ + Constructor; expects parameter d to be a dict of string parameters from a REST transport notification message + """ + self.signature = d['Signature'] # vH6ZbE0NhkF/hfNyxz2OgmzXYKs= + self.timestamp = d['Timestamp'] # 2006-05-23T23:22:30Z + self.version = d['Version'] # 2006-05-05 + assert d['method'] == NotificationMessage.OPERATION_NAME, "Method should be '%s'" % NotificationMessage.OPERATION_NAME + + # Build Events + self.events = [] + events_dict = {} + if 'Event' in d: + # TurboGears surprised me by 'doing the right thing' and making { 'Event': { '1': { 'EventType': ... } } } etc. + events_dict = d['Event'] + else: + for k in d: + v = d[k] + if k.startswith('Event.'): + ed = NotificationMessage.EVENT_RE.search(k).groupdict() + n = int(ed['n']) + param = str(ed['param']) + if n not in events_dict: + events_dict[n] = {} + events_dict[n][param] = v + for n in events_dict: + self.events.append(Event(events_dict[n])) + + def verify(self, secret_key): + """ + Verifies the authenticity of a notification message. + """ + verification_input = NotificationMessage.SERVICE_NAME + NotificationMessage.OPERATION_NAME + self.timestamp + signature_calc = self._auth_handler.sign_string(verification_input) + return self.signature == signature_calc + +class Event: + def __init__(self, d): + self.event_type = d['EventType'] + self.event_time_str = d['EventTime'] + self.hit_type = d['HITTypeId'] + self.hit_id = d['HITId'] + if 'AssignmentId' in d: # Not present in all event types + self.assignment_id = d['AssignmentId'] + + #TODO: build self.event_time datetime from string self.event_time_str + + def __repr__(self): + return "" % (self.event_type, self.hit_id) diff --git a/backup/src/boto/mturk/price.py b/backup/src/boto/mturk/price.py new file mode 100644 index 0000000..3c88a96 --- /dev/null +++ b/backup/src/boto/mturk/price.py @@ -0,0 +1,48 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Price: + + def __init__(self, amount=0.0, currency_code='USD'): + self.amount = amount + self.currency_code = currency_code + self.formatted_price = '' + + def __repr__(self): + if self.formatted_price: + return self.formatted_price + else: + return str(self.amount) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Amount': + self.amount = float(value) + elif name == 'CurrencyCode': + self.currency_code = value + elif name == 'FormattedPrice': + self.formatted_price = value + + def get_as_params(self, label, ord=1): + return {'%s.%d.Amount'%(label, ord) : str(self.amount), + '%s.%d.CurrencyCode'%(label, ord) : self.currency_code} diff --git a/backup/src/boto/mturk/qualification.py b/backup/src/boto/mturk/qualification.py new file mode 100644 index 0000000..6b620ec --- /dev/null +++ b/backup/src/boto/mturk/qualification.py @@ -0,0 +1,137 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Qualifications: + + def __init__(self, requirements=None): + if requirements == None: + requirements = [] + self.requirements = requirements + + def add(self, req): + self.requirements.append(req) + + def get_as_params(self): + params = {} + assert(len(self.requirements) <= 10) + for n, req in enumerate(self.requirements): + reqparams = req.get_as_params() + for rp in reqparams: + params['QualificationRequirement.%s.%s' % ((n+1),rp) ] = reqparams[rp] + return params + + +class Requirement(object): + """ + Representation of a single requirement + """ + + def __init__(self, qualification_type_id, comparator, integer_value=None, required_to_preview=False): + self.qualification_type_id = qualification_type_id + self.comparator = comparator + self.integer_value = integer_value + self.required_to_preview = required_to_preview + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + } + if self.comparator != 'Exists' and self.integer_value is not None: + params['IntegerValue'] = self.integer_value + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params + +class PercentAssignmentsSubmittedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000000", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsAbandonedRequirement(Requirement): + """ + The percentage of assignments the Worker has abandoned (allowed the deadline to elapse), over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000070", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsReturnedRequirement(Requirement): + """ + The percentage of assignments the Worker has returned, over all assignments the Worker has accepted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000E0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsApprovedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently approved by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000L0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class PercentAssignmentsRejectedRequirement(Requirement): + """ + The percentage of assignments the Worker has submitted that were subsequently rejected by the Requester, over all assignments the Worker has submitted. The value is an integer between 0 and 100. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="000000000000000000S0", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class NumberHitsApprovedRequirement(Requirement): + """ + Specifies the total number of HITs submitted by a Worker that have been approved. The value is an integer greater than or equal to 0. + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000040", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) + +class LocaleRequirement(Requirement): + """ + A Qualification requirement based on the Worker's location. The Worker's location is specified by the Worker to Mechanical Turk when the Worker creates his account. + """ + + def __init__(self, comparator, locale, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000071", comparator=comparator, integer_value=None, required_to_preview=required_to_preview) + self.locale = locale + + def get_as_params(self): + params = { + "QualificationTypeId": self.qualification_type_id, + "Comparator": self.comparator, + 'LocaleValue.Country': self.locale, + } + if self.required_to_preview: + params['RequiredToPreview'] = "true" + return params + +class AdultRequirement(Requirement): + """ + Requires workers to acknowledge that they are over 18 and that they agree to work on potentially offensive content. The value type is boolean, 1 (required), 0 (not required, the default). + """ + + def __init__(self, comparator, integer_value, required_to_preview=False): + Requirement.__init__(self, qualification_type_id="00000000000000000060", comparator=comparator, integer_value=integer_value, required_to_preview=required_to_preview) diff --git a/backup/src/boto/mturk/question.py b/backup/src/boto/mturk/question.py new file mode 100644 index 0000000..b1556ad --- /dev/null +++ b/backup/src/boto/mturk/question.py @@ -0,0 +1,396 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Question(object): + template = "%(items)s" + + def __init__(self, identifier, content, answer_spec, is_required=False, display_name=None): + # copy all of the parameters into object attributes + self.__dict__.update(vars()) + del self.self + + def get_as_params(self, label='Question'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + items = [ + SimpleField('QuestionIdentifier', self.identifier), + SimpleField('IsRequired', str(self.is_required).lower()), + self.content, + self.answer_spec, + ] + if self.display_name is not None: + items.insert(1, SimpleField('DisplayName', self.display_name)) + items = ''.join(item.get_as_xml() for item in items) + return self.template % vars() + +try: + from lxml import etree + class ValidatingXML(object): + def validate(self): + import urllib2 + schema_src_file = urllib2.urlopen(self.schema_url) + schema_doc = etree.parse(schema_src_file) + schema = etree.XMLSchema(schema_doc) + doc = etree.fromstring(self.get_as_xml()) + schema.assertValid(doc) +except ImportError: + class ValidatingXML(object): + def validate(self): pass + + +class ExternalQuestion(ValidatingXML): + """ + An object for constructing an External Question. + """ + schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/ExternalQuestion.xsd" + template = '%%(external_url)s%%(frame_height)s' % vars() + + def __init__(self, external_url, frame_height): + self.external_url = external_url + self.frame_height = frame_height + + def get_as_params(self, label='ExternalQuestion'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + return self.template % vars(self) + +class XMLTemplate: + def get_as_xml(self): + return self.template % vars(self) + +class SimpleField(object, XMLTemplate): + """ + A Simple name/value pair that can be easily rendered as XML. + + >>> SimpleField('Text', 'A text string').get_as_xml() + 'A text string' + """ + template = '<%(field)s>%(value)s' + + def __init__(self, field, value): + self.field = field + self.value = value + +class Binary(object, XMLTemplate): + template = """%(type)s%(subtype)s%(url)s%(alt_text)s""" + def __init__(self, type, subtype, url, alt_text): + self.__dict__.update(vars()) + del self.self + +class List(list): + """A bulleted list suitable for OrderedContent or Overview content""" + def get_as_xml(self): + items = ''.join('%s' % item for item in self) + return '%s' % items + +class Application(object): + template = "<%(class_)s>%(content)s" + parameter_template = "%(name)s%(value)s" + + def __init__(self, width, height, **parameters): + self.width = width + self.height = height + self.parameters = parameters + + def get_inner_content(self, content): + content.append_field('Width', self.width) + content.append_field('Height', self.height) + for name, value in self.parameters.items(): + value = self.parameter_template % vars() + content.append_field('ApplicationParameter', value) + + def get_as_xml(self): + content = OrderedContent() + self.get_inner_content(content) + content = content.get_as_xml() + class_ = self.__class__.__name__ + return self.template % vars() + +class JavaApplet(Application): + def __init__(self, path, filename, *args, **kwargs): + self.path = path + self.filename = filename + super(JavaApplet, self).__init__(*args, **kwargs) + + def get_inner_content(self, content): + content = OrderedContent() + content.append_field('AppletPath', self.path) + content.append_field('AppletFilename', self.filename) + super(JavaApplet, self).get_inner_content(content) + +class Flash(Application): + def __init__(self, url, *args, **kwargs): + self.url = url + super(Flash, self).__init__(*args, **kwargs) + + def get_inner_content(self, content): + content = OrderedContent() + content.append_field('FlashMovieURL', self.url) + super(Flash, self).get_inner_content(content) + +class FormattedContent(object, XMLTemplate): + schema_url = 'http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2006-07-14/FormattedContentXHTMLSubset.xsd' + template = '' + def __init__(self, content): + self.content = content + +class OrderedContent(list): + + def append_field(self, field, value): + self.append(SimpleField(field, value)) + + def get_as_xml(self): + return ''.join(item.get_as_xml() for item in self) + +class Overview(OrderedContent): + template = '%(content)s' + + def get_as_params(self, label='Overview'): + return { label : self.get_as_xml() } + + def get_as_xml(self): + content = super(Overview, self).get_as_xml() + return self.template % vars() + +class QuestionForm(ValidatingXML, list): + """ + From the AMT API docs: + + The top-most element of the QuestionForm data structure is a QuestionForm element. This + element contains optional Overview elements and one or more Question elements. There can be + any number of these two element types listed in any order. The following example structure has an + Overview element and a Question element followed by a second Overview element and Question + element--all within the same QuestionForm. + + + + [...] + + + [...] + + + [...] + + + [...] + + [...] + + + QuestionForm is implemented as a list, so to construct a + QuestionForm, simply append Questions and Overviews (with at least + one Question). + """ + schema_url = "http://mechanicalturk.amazonaws.com/AWSMechanicalTurkDataSchemas/2005-10-01/QuestionForm.xsd" + xml_template = """%%(items)s""" % vars() + + def is_valid(self): + return ( + any(isinstance(item, Question) for item in self) + and + all(isinstance(item, (Question, Overview)) for item in self) + ) + + def get_as_xml(self): + assert self.is_valid(), "QuestionForm contains invalid elements" + items = ''.join(item.get_as_xml() for item in self) + return self.xml_template % vars() + +class QuestionContent(OrderedContent): + template = '%(content)s' + + def get_as_xml(self): + content = super(QuestionContent, self).get_as_xml() + return self.template % vars() + +class AnswerSpecification(object): + template = '%(spec)s' + + def __init__(self, spec): + self.spec = spec + + def get_as_xml(self): + spec = self.spec.get_as_xml() + return self.template % vars() + +class Constraints(OrderedContent): + template = '%(content)s' + + def get_as_xml(self): + content = super(Constraints, self).get_as_xml() + return self.template % vars() + +class Constraint(object): + def get_attributes(self): + pairs = zip(self.attribute_names, self.attribute_values) + attrs = ' '.join( + '%s="%d"' % (name,value) + for (name,value) in pairs + if value is not None + ) + return attrs + + def get_as_xml(self): + attrs = self.get_attributes() + return self.template % vars() + +class NumericConstraint(Constraint): + attribute_names = 'minValue', 'maxValue' + template = '' + + def __init__(self, min_value=None, max_value=None): + self.attribute_values = min_value, max_value + +class LengthConstraint(Constraint): + attribute_names = 'minLength', 'maxLength' + template = '' + + def __init__(self, min_length=None, max_length=None): + self.attribute_values = min_length, max_length + +class RegExConstraint(Constraint): + attribute_names = 'regex', 'errorText', 'flags' + template = '' + + def __init__(self, pattern, error_text=None, flags=None): + self.attribute_values = pattern, error_text, flags + +class NumberOfLinesSuggestion(object): + template = '%(num_lines)s' + + def __init__(self, num_lines=1): + self.num_lines = num_lines + + def get_as_xml(self): + num_lines = self.num_lines + return self.template % vars() + +class FreeTextAnswer(object): + template = '%(items)s' + + def __init__(self, default=None, constraints=None, num_lines=None): + self.default = default + if constraints is None: constraints = Constraints() + self.constraints = Constraints(constraints) + self.num_lines = num_lines + + def get_as_xml(self): + constraints = Constraints() + items = [constraints] + if self.default: + items.append(SimpleField('DefaultText', self.default)) + if self.num_lines: + items.append(NumberOfLinesSuggestion(self.num_lines)) + items = ''.join(item.get_as_xml() for item in items) + return self.template % vars() + +class FileUploadAnswer(object): + template = """%(min_bytes)d%(max_bytes)d""" + + def __init__(self, min_bytes, max_bytes): + assert 0 <= min_bytes <= max_bytes <= 2*10**9 + self.min_bytes = min_bytes + self.max_bytes = max_bytes + + def get_as_xml(self): + return self.template % vars(self) + +class SelectionAnswer(object): + """ + A class to generate SelectionAnswer XML data structures. + Does not yet implement Binary selection options. + """ + SELECTIONANSWER_XML_TEMPLATE = """%s%s%s""" # % (count_xml, style_xml, selections_xml) + SELECTION_XML_TEMPLATE = """%s%s""" # (identifier, value_xml) + SELECTION_VALUE_XML_TEMPLATE = """<%s>%s""" # (type, value, type) + STYLE_XML_TEMPLATE = """%s""" # (style) + MIN_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + MAX_SELECTION_COUNT_XML_TEMPLATE = """%s""" # count + ACCEPTED_STYLES = ['radiobutton', 'dropdown', 'checkbox', 'list', 'combobox', 'multichooser'] + OTHER_SELECTION_ELEMENT_NAME = 'OtherSelection' + + def __init__(self, min=1, max=1, style=None, selections=None, type='text', other=False): + + if style is not None: + if style in SelectionAnswer.ACCEPTED_STYLES: + self.style_suggestion = style + else: + raise ValueError("style '%s' not recognized; should be one of %s" % (style, ', '.join(SelectionAnswer.ACCEPTED_STYLES))) + else: + self.style_suggestion = None + + if selections is None: + raise ValueError("SelectionAnswer.__init__(): selections must be a non-empty list of (content, identifier) tuples") + else: + self.selections = selections + + self.min_selections = min + self.max_selections = max + + assert len(selections) >= self.min_selections, "# of selections is less than minimum of %d" % self.min_selections + #assert len(selections) <= self.max_selections, "# of selections exceeds maximum of %d" % self.max_selections + + self.type = type + + self.other = other + + def get_as_xml(self): + if self.type == 'text': + TYPE_TAG = "Text" + elif self.type == 'binary': + TYPE_TAG = "Binary" + else: + raise ValueError("illegal type: %s; must be either 'text' or 'binary'" % str(self.type)) + + # build list of elements + selections_xml = "" + for tpl in self.selections: + value_xml = SelectionAnswer.SELECTION_VALUE_XML_TEMPLATE % (TYPE_TAG, tpl[0], TYPE_TAG) + selection_xml = SelectionAnswer.SELECTION_XML_TEMPLATE % (tpl[1], value_xml) + selections_xml += selection_xml + + if self.other: + # add OtherSelection element as xml if available + if hasattr(self.other, 'get_as_xml'): + assert type(self.other) == FreeTextAnswer, 'OtherSelection can only be a FreeTextAnswer' + selections_xml += self.other.get_as_xml().replace('FreeTextAnswer', 'OtherSelection') + else: + selections_xml += "" + + if self.style_suggestion is not None: + style_xml = SelectionAnswer.STYLE_XML_TEMPLATE % self.style_suggestion + else: + style_xml = "" + + if self.style_suggestion != 'radiobutton': + count_xml = SelectionAnswer.MIN_SELECTION_COUNT_XML_TEMPLATE %self.min_selections + count_xml += SelectionAnswer.MAX_SELECTION_COUNT_XML_TEMPLATE %self.max_selections + else: + count_xml = "" + + ret = SelectionAnswer.SELECTIONANSWER_XML_TEMPLATE % (count_xml, style_xml, selections_xml) + + # return XML + return ret + diff --git a/backup/src/boto/mturk/test/.gitignore b/backup/src/boto/mturk/test/.gitignore new file mode 100644 index 0000000..039e4d4 --- /dev/null +++ b/backup/src/boto/mturk/test/.gitignore @@ -0,0 +1 @@ +local.py diff --git a/backup/src/boto/mturk/test/__init__.py b/backup/src/boto/mturk/test/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backup/src/boto/mturk/test/_init_environment.py b/backup/src/boto/mturk/test/_init_environment.py new file mode 100644 index 0000000..ec026ec --- /dev/null +++ b/backup/src/boto/mturk/test/_init_environment.py @@ -0,0 +1,24 @@ +import os +import functools + +live_connection = False +mturk_host = 'mechanicalturk.sandbox.amazonaws.com' +external_url = 'http://www.example.com/' + +try: + local = os.path.join(os.path.dirname(__file__), 'local.py') + execfile(local) +except: + pass + +if live_connection: + #TODO: you must set the auth credentials to something valid + from boto.mturk.connection import MTurkConnection +else: + # Here the credentials must be set, but it doesn't matter what + # they're set to. + os.environ.setdefault('AWS_ACCESS_KEY_ID', 'foo') + os.environ.setdefault('AWS_SECRET_ACCESS_KEY', 'bar') + from mocks import MTurkConnection + +SetHostMTurkConnection = functools.partial(MTurkConnection, host=mturk_host) diff --git a/backup/src/boto/mturk/test/all_tests.py b/backup/src/boto/mturk/test/all_tests.py new file mode 100644 index 0000000..f17cf85 --- /dev/null +++ b/backup/src/boto/mturk/test/all_tests.py @@ -0,0 +1,24 @@ + +import unittest +import doctest +from glob import glob + +from create_hit_test import * +from create_hit_with_qualifications import * +from create_hit_external import * +from create_hit_with_qualifications import * +from hit_persistence import * + +doctest_suite = doctest.DocFileSuite( + *glob('*.doctest'), + optionflags=doctest.REPORT_ONLY_FIRST_FAILURE + ) + +class Program(unittest.TestProgram): + def runTests(self, *args, **kwargs): + self.test = unittest.TestSuite([self.test, doctest_suite]) + super(Program, self).runTests(*args, **kwargs) + +if __name__ == '__main__': + Program() + diff --git a/backup/src/boto/mturk/test/cleanup_tests.py b/backup/src/boto/mturk/test/cleanup_tests.py new file mode 100644 index 0000000..2381dd9 --- /dev/null +++ b/backup/src/boto/mturk/test/cleanup_tests.py @@ -0,0 +1,45 @@ +import itertools + +from _init_environment import SetHostMTurkConnection + +def description_filter(substring): + return lambda hit: substring in hit.Title + +def disable_hit(hit): + return conn.disable_hit(hit.HITId) + +def dispose_hit(hit): + # assignments must be first approved or rejected + for assignment in conn.get_assignments(hit.HITId): + if assignment.AssignmentStatus == 'Submitted': + conn.approve_assignment(assignment.AssignmentId) + return conn.dispose_hit(hit.HITId) + +def cleanup(): + """Remove any boto test related HIT's""" + + global conn + + conn = SetHostMTurkConnection() + + + is_boto = description_filter('Boto') + print 'getting hits...' + all_hits = list(conn.get_all_hits()) + is_reviewable = lambda hit: hit.HITStatus == 'Reviewable' + is_not_reviewable = lambda hit: not is_reviewable(hit) + hits_to_process = filter(is_boto, all_hits) + hits_to_disable = filter(is_not_reviewable, hits_to_process) + hits_to_dispose = filter(is_reviewable, hits_to_process) + print 'disabling/disposing %d/%d hits' % (len(hits_to_disable), len(hits_to_dispose)) + map(disable_hit, hits_to_disable) + map(dispose_hit, hits_to_dispose) + + total_hits = len(all_hits) + hits_processed = len(hits_to_process) + skipped = total_hits - hits_processed + fmt = 'Processed: %(total_hits)d HITs, disabled/disposed: %(hits_processed)d, skipped: %(skipped)d' + print fmt % vars() + +if __name__ == '__main__': + cleanup() diff --git a/backup/src/boto/mturk/test/common.py b/backup/src/boto/mturk/test/common.py new file mode 100644 index 0000000..23361bd --- /dev/null +++ b/backup/src/boto/mturk/test/common.py @@ -0,0 +1,44 @@ +import unittest +import uuid +import datetime + +from boto.mturk.question import ( + Question, QuestionContent, AnswerSpecification, FreeTextAnswer, +) +from _init_environment import SetHostMTurkConnection + +class MTurkCommon(unittest.TestCase): + def setUp(self): + self.conn = SetHostMTurkConnection() + + @staticmethod + def get_question(): + # create content for a question + qn_content = QuestionContent() + qn_content.append_field('Title', 'Boto no hit type question content') + qn_content.append_field('Text', 'What is a boto no hit type?') + + # create the question specification + qn = Question(identifier=str(uuid.uuid4()), + content=qn_content, + answer_spec=AnswerSpecification(FreeTextAnswer())) + return qn + + @staticmethod + def get_hit_params(): + return dict( + lifetime=datetime.timedelta(minutes=65), + max_assignments=2, + title='Boto create_hit title', + description='Boto create_hit description', + keywords=['boto', 'test'], + reward=0.23, + duration=datetime.timedelta(minutes=6), + approval_delay=60*60, + annotation='An annotation from boto create_hit test', + response_groups=['Minimal', + 'HITDetail', + 'HITQuestion', + 'HITAssignmentSummary',], + ) + diff --git a/backup/src/boto/mturk/test/create_free_text_question_regex.doctest b/backup/src/boto/mturk/test/create_free_text_question_regex.doctest new file mode 100644 index 0000000..0b9d2a9 --- /dev/null +++ b/backup/src/boto/mturk/test/create_free_text_question_regex.doctest @@ -0,0 +1,100 @@ +>>> import uuid +>>> import datetime +>>> from _init_environment import MTurkConnection, mturk_host +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, RegExConstraint + +>>> conn = MTurkConnection(host=mturk_host) + +# create content for a question +>>> qn_content = QuestionContent() +>>> qn_content.append_field('Title', 'Boto no hit type question content') +>>> qn_content.append_field('Text', 'What is a boto no hit type?') + +# create a free text answer that is not quite so free! +>>> constraints = [ +... RegExConstraint( +... "^[12][0-9]{3}-[01]?\d-[0-3]?\d$", +... error_text="You must enter a date with the format yyyy-mm-dd.", +... flags='i', +... )] +>>> ft_answer = FreeTextAnswer(constraints=constraints, +... default="This is not a valid format") + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(ft_answer)) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> len(create_hit_rs) +1 +>>> hit = create_hit_rs[0] +>>> hit_type_id = hit.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> hit.MaxAssignments +u'2' + +>>> hit.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> abs(delta).seconds < 5 +True + +# duration is as specified for the HIT type +>>> hit.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(float(hit.Amount) * 100) +23 + +>>> hit.FormattedPrice +u'$0.23' + +# only US currency supported at present +>>> hit.CurrencyCode +u'USD' + +# title is the HIT type title +>>> hit.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> hit.Description +u'Boto create_hit description' + +# annotation is correct +>>> hit.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> hit.HITReviewStatus +u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit.doctest b/backup/src/boto/mturk/test/create_hit.doctest new file mode 100644 index 0000000..a97cbf8 --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit.doctest @@ -0,0 +1,92 @@ +>>> import uuid +>>> import datetime +>>> from _init_environment import MTurkConnection, mturk_host +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host=mturk_host) + +# create content for a question +>>> qn_content = QuestionContent() +>>> qn_content.append_field('Title', 'Boto no hit type question content') +>>> qn_content.append_field('Text', 'What is a boto no hit type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> lifetime = datetime.timedelta(minutes=65) +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=lifetime, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +>>> len(create_hit_rs) +1 +>>> hit = create_hit_rs[0] + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = hit.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> hit.MaxAssignments +u'2' + +>>> hit.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime +>>> expected_datetime = datetime.datetime.utcnow() + lifetime +>>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> abs(delta).seconds < 5 +True + +# duration is as specified for the HIT type +>>> hit.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(float(hit.Amount) * 100) +23 + +>>> hit.FormattedPrice +u'$0.23' + +# only US currency supported at present +>>> hit.CurrencyCode +u'USD' + +# title is the HIT type title +>>> hit.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> hit.Description +u'Boto create_hit description' + +# annotation is correct +>>> hit.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> hit.HITReviewStatus +u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_binary.doctest b/backup/src/boto/mturk/test/create_hit_binary.doctest new file mode 100644 index 0000000..3f0434e --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit_binary.doctest @@ -0,0 +1,94 @@ +>>> import uuid +>>> import datetime +>>> from _init_environment import MTurkConnection, mturk_host +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer, Binary + +>>> conn = MTurkConnection(host=mturk_host) + +# create content for a question +>>> qn_content = QuestionContent() +>>> qn_content.append_field('Title','Boto no hit type question content') +>>> qn_content.append_field('Text', 'What is a boto binary hit type?') +>>> binary_content = Binary('image', 'jpeg', 'http://www.example.com/test1.jpg', alt_text='image is missing') +>>> qn_content.append(binary_content) + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> lifetime = datetime.timedelta(minutes=65) +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=lifetime, +... max_assignments=2, +... title='Boto create_hit title', +... description='Boto create_hit description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +>>> len(create_hit_rs) +1 +>>> hit = create_hit_rs[0] + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = hit.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> hit.MaxAssignments +u'2' + +>>> hit.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime +>>> expected_datetime = datetime.datetime.utcnow() + lifetime +>>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> abs(delta).seconds < 5 +True + +# duration is as specified for the HIT type +>>> hit.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly (allow for float error here) +>>> int(float(hit.Amount) * 100) +23 + +>>> hit.FormattedPrice +u'$0.23' + +# only US currency supported at present +>>> hit.CurrencyCode +u'USD' + +# title is the HIT type title +>>> hit.Title +u'Boto create_hit title' + +# title is the HIT type description +>>> hit.Description +u'Boto create_hit description' + +# annotation is correct +>>> hit.RequesterAnnotation +u'An annotation from boto create_hit test' + +>>> hit.HITReviewStatus +u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_external.py b/backup/src/boto/mturk/test/create_hit_external.py new file mode 100644 index 0000000..9e955a6 --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit_external.py @@ -0,0 +1,17 @@ +import unittest +import uuid +import datetime +from boto.mturk.question import ExternalQuestion + +from _init_environment import SetHostMTurkConnection, external_url + +class Test(unittest.TestCase): + def test_create_hit_external(self): + q = ExternalQuestion(external_url=external_url, frame_height=800) + conn = SetHostMTurkConnection() + keywords=['boto', 'test', 'doctest'] + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',]) + assert(create_hit_rs.status == True) + +if __name__ == "__main__": + unittest.main() diff --git a/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest b/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest new file mode 100644 index 0000000..1b6d0f0 --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit_from_hit_type.doctest @@ -0,0 +1,103 @@ +>>> import uuid +>>> import datetime +>>> from _init_environment import MTurkConnection, mturk_host +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer +>>> +>>> conn = MTurkConnection(host=mturk_host) +>>> keywords=['boto', 'test', 'doctest'] +>>> hit_type_rs = conn.register_hit_type('Boto Test HIT type', +... 'HIT Type for testing Boto', +... 0.12, +... 60*6, +... keywords=keywords, +... approval_delay=60*60) + +# this was a valid request +>>> hit_type_rs.status +True + +# the HIT Type Id is a unicode string +>>> hit_type_id = hit_type_rs.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +# create content for a question +>>> qn_content = QuestionContent() +>>> qn_content.append_field('Title', 'Boto question content create_hit_from_hit_type') +>>> qn_content.append_field('Text', 'What is a boto create_hit_from_hit_type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question using the HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> create_hit_rs = conn.create_hit(hit_type=hit_type_rs.HITTypeId, +... question=qn, +... lifetime=60*65, +... max_assignments=2, +... annotation='An annotation from boto create_hit_from_hit_type test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +>>> len(create_hit_rs) +1 + +>>> hit = create_hit_rs[0] + +# for the requested hit type id +>>> hit.HITTypeId == hit_type_id +True + +# with the correct number of maximum assignments +>>> hit.MaxAssignments +u'2' + +# and the approval delay +>>> hit.AutoApprovalDelayInSeconds +u'3600' + +# expiration should be very close to now + the lifetime in seconds +>>> expected_datetime = datetime.datetime.utcnow() + datetime.timedelta(seconds=3900) +>>> expiration_datetime = datetime.datetime.strptime(hit.Expiration, '%Y-%m-%dT%H:%M:%SZ') +>>> delta = expected_datetime - expiration_datetime +>>> abs(delta).seconds < 5 +True + +# duration is as specified for the HIT type +>>> hit.AssignmentDurationInSeconds +u'360' + +# the reward has been set correctly +>>> float(hit.Amount) == 0.12 +True + +>>> hit.FormattedPrice +u'$0.12' + +# only US currency supported at present +>>> hit.CurrencyCode +u'USD' + +# title is the HIT type title +>>> hit.Title +u'Boto Test HIT type' + +# title is the HIT type description +>>> hit.Description +u'HIT Type for testing Boto' + +# annotation is correct +>>> hit.RequesterAnnotation +u'An annotation from boto create_hit_from_hit_type test' + +# not reviewed yet +>>> hit.HITReviewStatus +u'NotReviewed' diff --git a/backup/src/boto/mturk/test/create_hit_test.py b/backup/src/boto/mturk/test/create_hit_test.py new file mode 100644 index 0000000..a690d80 --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit_test.py @@ -0,0 +1,21 @@ +import unittest +import os +from boto.mturk.question import QuestionForm + +from common import MTurkCommon + +class TestHITCreation(MTurkCommon): + def testCallCreateHitWithOneQuestion(self): + create_hit_rs = self.conn.create_hit( + question=self.get_question(), + **self.get_hit_params() + ) + + def testCallCreateHitWithQuestionForm(self): + create_hit_rs = self.conn.create_hit( + questions=QuestionForm([self.get_question()]), + **self.get_hit_params() + ) + +if __name__ == '__main__': + unittest.main() diff --git a/backup/src/boto/mturk/test/create_hit_with_qualifications.py b/backup/src/boto/mturk/test/create_hit_with_qualifications.py new file mode 100644 index 0000000..9ef2bc5 --- /dev/null +++ b/backup/src/boto/mturk/test/create_hit_with_qualifications.py @@ -0,0 +1,16 @@ +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import ExternalQuestion +from boto.mturk.qualification import Qualifications, PercentAssignmentsApprovedRequirement + +def test(): + q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800) + conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') + keywords=['boto', 'test', 'doctest'] + qualifications = Qualifications() + qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")) + create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications) + assert(create_hit_rs.status == True) + print create_hit_rs.HITTypeId + +if __name__ == "__main__": + test() diff --git a/backup/src/boto/mturk/test/hit_persistence.py b/backup/src/boto/mturk/test/hit_persistence.py new file mode 100644 index 0000000..6991856 --- /dev/null +++ b/backup/src/boto/mturk/test/hit_persistence.py @@ -0,0 +1,27 @@ +import unittest +import pickle + +from common import MTurkCommon + +class TestHITPersistence(MTurkCommon): + def create_hit_result(self): + return self.conn.create_hit( + question=self.get_question(), **self.get_hit_params() + ) + + def test_pickle_hit_result(self): + result = self.create_hit_result() + new_result = pickle.loads(pickle.dumps(result)) + + def test_pickle_deserialized_version(self): + """ + It seems the technique used to store and reload the object must + result in an equivalent object, or subsequent pickles may fail. + This tests a double-pickle to elicit that error. + """ + result = self.create_hit_result() + new_result = pickle.loads(pickle.dumps(result)) + pickle.dumps(new_result) + +if __name__ == '__main__': + unittest.main() diff --git a/backup/src/boto/mturk/test/mocks.py b/backup/src/boto/mturk/test/mocks.py new file mode 100644 index 0000000..d3f0f2e --- /dev/null +++ b/backup/src/boto/mturk/test/mocks.py @@ -0,0 +1,11 @@ +from boto.mturk.connection import MTurkConnection as RealMTurkConnection + +class MTurkConnection(RealMTurkConnection): + """ + Mock MTurkConnection that doesn't connect, but instead just prepares + the request and captures information about its usage. + """ + + def _process_request(self, *args, **kwargs): + saved_args = self.__dict__.setdefault('_mock_saved_args', dict()) + saved_args['_process_request'] = (args, kwargs) diff --git a/backup/src/boto/mturk/test/reviewable_hits.doctest b/backup/src/boto/mturk/test/reviewable_hits.doctest new file mode 100644 index 0000000..113a056 --- /dev/null +++ b/backup/src/boto/mturk/test/reviewable_hits.doctest @@ -0,0 +1,129 @@ +>>> import uuid +>>> import datetime +>>> from _init_environment import MTurkConnection, mturk_host +>>> from boto.mturk.question import Question, QuestionContent, AnswerSpecification, FreeTextAnswer + +>>> conn = MTurkConnection(host=mturk_host) + +# create content for a question +>>> qn_content = QuestionContent() +>>> qn_content.append_field('Title', 'Boto no hit type question content') +>>> qn_content.append_field('Text', 'What is a boto no hit type?') + +# create the question specification +>>> qn = Question(identifier=str(uuid.uuid4()), +... content=qn_content, +... answer_spec=AnswerSpecification(FreeTextAnswer())) + +# now, create the actual HIT for the question without using a HIT type +# NOTE - the response_groups are specified to get back additional information for testing +>>> keywords=['boto', 'test', 'doctest'] +>>> create_hit_rs = conn.create_hit(question=qn, +... lifetime=60*65, +... max_assignments=1, +... title='Boto Hit to be Reviewed', +... description='Boto reviewable_hits description', +... keywords=keywords, +... reward=0.23, +... duration=60*6, +... approval_delay=60*60, +... annotation='An annotation from boto create_hit test', +... response_groups=['Minimal', +... 'HITDetail', +... 'HITQuestion', +... 'HITAssignmentSummary',]) + +# this is a valid request +>>> create_hit_rs.status +True + +>>> len(create_hit_rs) +1 +>>> hit = create_hit_rs[0] + +# for the requested hit type id +# the HIT Type Id is a unicode string +>>> hit_type_id = hit.HITTypeId +>>> hit_type_id # doctest: +ELLIPSIS +u'...' + +>>> from selenium_support import complete_hit, has_selenium +>>> if has_selenium(): complete_hit(hit_type_id, response='reviewable_hits_test') +>>> import time + +Give mechanical turk some time to process the hit +>>> if has_selenium(): time.sleep(10) + +# should have some reviewable HIT's returned, especially if returning all HIT type's +# NOTE: but only if your account has existing HIT's in the reviewable state +>>> reviewable_rs = conn.get_reviewable_hits() + +# this is a valid request +>>> reviewable_rs.status +True + +>>> len(reviewable_rs) >= 1 +True + +# should contain at least one HIT object +>>> reviewable_rs # doctest: +ELLIPSIS +[>> hit_id = reviewable_rs[0].HITId + +# check that we can retrieve the assignments for a HIT +>>> assignments_rs = conn.get_assignments(hit_id) + +# this is a valid request +>>> assignments_rs.status +True + +>>> int(assignments_rs.NumResults) >= 1 +True + +>>> len(assignments_rs) == int(assignments_rs.NumResults) +True + +>>> assignments_rs.PageNumber +u'1' + +>>> assignments_rs.TotalNumResults >= 1 +True + +# should contain at least one Assignment object +>>> assignments_rs # doctest: +ELLIPSIS +[>> assignment = assignments_rs[0] + +>>> assignment.HITId == hit_id +True + +# should have a valid status +>>> assignment.AssignmentStatus in ['Submitted', 'Approved', 'Rejected'] +True + +# should have returned at least one answer +>>> len(assignment.answers) > 0 +True + +# should contain at least one set of QuestionFormAnswer objects +>>> assignment.answers # doctest: +ELLIPSIS +[[>> answer = assignment.answers[0][0] + +# the answer should have exactly one field +>>> len(answer.fields) +1 + +>>> qid, text = answer.fields[0] + +>>> text # doctest: +ELLIPSIS +u'...' + +# question identifier should be a unicode string +>>> qid # doctest: +ELLIPSIS +u'...' + diff --git a/backup/src/boto/mturk/test/run-doctest.py b/backup/src/boto/mturk/test/run-doctest.py new file mode 100644 index 0000000..f10c762 --- /dev/null +++ b/backup/src/boto/mturk/test/run-doctest.py @@ -0,0 +1,15 @@ +from __future__ import print_function + +import argparse +import doctest + +parser = argparse.ArgumentParser( + description="Run a test by name" + ) +parser.add_argument('test_name') +args = parser.parse_args() + +doctest.testfile( + args.test_name, + optionflags=doctest.REPORT_ONLY_FIRST_FAILURE + ) \ No newline at end of file diff --git a/backup/src/boto/mturk/test/search_hits.doctest b/backup/src/boto/mturk/test/search_hits.doctest new file mode 100644 index 0000000..a79bab7 --- /dev/null +++ b/backup/src/boto/mturk/test/search_hits.doctest @@ -0,0 +1,16 @@ +>>> from _init_environment import MTurkConnection, mturk_host +>>> conn = MTurkConnection(host=mturk_host) + +# should have some HIT's returned by a search (but only if your account has existing HIT's) +>>> search_rs = conn.search_hits() + +# this is a valid request +>>> search_rs.status +True + +>>> len(search_rs) > 1 +True + +>>> search_rs # doctest: +ELLIPSIS +[= (2,7): + import unittest +else: + import unittest2 as unittest diff --git a/backup/src/boto/mturk/test/test_disable_hit.py b/backup/src/boto/mturk/test/test_disable_hit.py new file mode 100644 index 0000000..2e2701d --- /dev/null +++ b/backup/src/boto/mturk/test/test_disable_hit.py @@ -0,0 +1,11 @@ +from boto.mturk.test.support import unittest + +from common import MTurkCommon +from boto.mturk.connection import MTurkRequestError + +class TestDisableHITs(MTurkCommon): + def test_disable_invalid_hit(self): + self.assertRaises(MTurkRequestError, self.conn.disable_hit, 'foo') + +if __name__ == '__main__': + unittest.main() diff --git a/backup/src/boto/plugin.py b/backup/src/boto/plugin.py new file mode 100644 index 0000000..f8b592c --- /dev/null +++ b/backup/src/boto/plugin.py @@ -0,0 +1,90 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +""" +Implements plugin related api. + +To define a new plugin just subclass Plugin, like this. + +class AuthPlugin(Plugin): + pass + +Then start creating subclasses of your new plugin. + +class MyFancyAuth(AuthPlugin): + capability = ['sign', 'vmac'] + +The actual interface is duck typed. + +""" + +import glob +import imp, os.path + +class Plugin(object): + """Base class for all plugins.""" + + capability = [] + + @classmethod + def is_capable(cls, requested_capability): + """Returns true if the requested capability is supported by this plugin + """ + for c in requested_capability: + if not c in cls.capability: + return False + return True + +def get_plugin(cls, requested_capability=None): + if not requested_capability: + requested_capability = [] + result = [] + for handler in cls.__subclasses__(): + if handler.is_capable(requested_capability): + result.append(handler) + return result + +def _import_module(filename): + (path, name) = os.path.split(filename) + (name, ext) = os.path.splitext(name) + + (file, filename, data) = imp.find_module(name, [path]) + try: + return imp.load_module(name, file, filename, data) + finally: + if file: + file.close() + +_plugin_loaded = False + +def load_plugins(config): + global _plugin_loaded + if _plugin_loaded: + return + _plugin_loaded = True + + if not config.has_option('Plugin', 'plugin_directory'): + return + directory = config.get('Plugin', 'plugin_directory') + for file in glob.glob(os.path.join(directory, '*.py')): + _import_module(file) + diff --git a/backup/src/boto/provider.py b/backup/src/boto/provider.py new file mode 100644 index 0000000..c1c8b59 --- /dev/null +++ b/backup/src/boto/provider.py @@ -0,0 +1,208 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright 2010 Google Inc. +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +""" +This class encapsulates the provider-specific header differences. +""" + +import os +import boto +from boto import config +from boto.gs.acl import ACL +from boto.gs.acl import CannedACLStrings as CannedGSACLStrings +from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings +from boto.s3.acl import Policy + +HEADER_PREFIX_KEY = 'header_prefix' +METADATA_PREFIX_KEY = 'metadata_prefix' + +AWS_HEADER_PREFIX = 'x-amz-' +GOOG_HEADER_PREFIX = 'x-goog-' + +ACL_HEADER_KEY = 'acl-header' +AUTH_HEADER_KEY = 'auth-header' +COPY_SOURCE_HEADER_KEY = 'copy-source-header' +COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header' +DELETE_MARKER_HEADER_KEY = 'delete-marker-header' +DATE_HEADER_KEY = 'date-header' +METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header' +RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header' +SECURITY_TOKEN_HEADER_KEY = 'security-token-header' +STORAGE_CLASS_HEADER_KEY = 'storage-class' +MFA_HEADER_KEY = 'mfa-header' +VERSION_ID_HEADER_KEY = 'version-id-header' + +STORAGE_COPY_ERROR = 'StorageCopyError' +STORAGE_CREATE_ERROR = 'StorageCreateError' +STORAGE_DATA_ERROR = 'StorageDataError' +STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError' +STORAGE_RESPONSE_ERROR = 'StorageResponseError' + + +class Provider(object): + + CredentialMap = { + 'aws' : ('aws_access_key_id', 'aws_secret_access_key'), + 'google' : ('gs_access_key_id', 'gs_secret_access_key'), + } + + AclClassMap = { + 'aws' : Policy, + 'google' : ACL + } + + CannedAclsMap = { + 'aws' : CannedS3ACLStrings, + 'google' : CannedGSACLStrings + } + + HostKeyMap = { + 'aws' : 's3', + 'google' : 'gs' + } + + HeaderInfoMap = { + 'aws' : { + HEADER_PREFIX_KEY : AWS_HEADER_PREFIX, + METADATA_PREFIX_KEY : AWS_HEADER_PREFIX + 'meta-', + ACL_HEADER_KEY : AWS_HEADER_PREFIX + 'acl', + AUTH_HEADER_KEY : 'AWS', + COPY_SOURCE_HEADER_KEY : AWS_HEADER_PREFIX + 'copy-source', + COPY_SOURCE_VERSION_ID_HEADER_KEY : AWS_HEADER_PREFIX + + 'copy-source-version-id', + DATE_HEADER_KEY : AWS_HEADER_PREFIX + 'date', + DELETE_MARKER_HEADER_KEY : AWS_HEADER_PREFIX + 'delete-marker', + METADATA_DIRECTIVE_HEADER_KEY : AWS_HEADER_PREFIX + + 'metadata-directive', + RESUMABLE_UPLOAD_HEADER_KEY : None, + SECURITY_TOKEN_HEADER_KEY : AWS_HEADER_PREFIX + 'security-token', + VERSION_ID_HEADER_KEY : AWS_HEADER_PREFIX + 'version-id', + STORAGE_CLASS_HEADER_KEY : AWS_HEADER_PREFIX + 'storage-class', + MFA_HEADER_KEY : AWS_HEADER_PREFIX + 'mfa', + }, + 'google' : { + HEADER_PREFIX_KEY : GOOG_HEADER_PREFIX, + METADATA_PREFIX_KEY : GOOG_HEADER_PREFIX + 'meta-', + ACL_HEADER_KEY : GOOG_HEADER_PREFIX + 'acl', + AUTH_HEADER_KEY : 'GOOG1', + COPY_SOURCE_HEADER_KEY : GOOG_HEADER_PREFIX + 'copy-source', + COPY_SOURCE_VERSION_ID_HEADER_KEY : GOOG_HEADER_PREFIX + + 'copy-source-version-id', + DATE_HEADER_KEY : GOOG_HEADER_PREFIX + 'date', + DELETE_MARKER_HEADER_KEY : GOOG_HEADER_PREFIX + 'delete-marker', + METADATA_DIRECTIVE_HEADER_KEY : GOOG_HEADER_PREFIX + + 'metadata-directive', + RESUMABLE_UPLOAD_HEADER_KEY : GOOG_HEADER_PREFIX + 'resumable', + SECURITY_TOKEN_HEADER_KEY : GOOG_HEADER_PREFIX + 'security-token', + VERSION_ID_HEADER_KEY : GOOG_HEADER_PREFIX + 'version-id', + STORAGE_CLASS_HEADER_KEY : None, + MFA_HEADER_KEY : None, + } + } + + ErrorMap = { + 'aws' : { + STORAGE_COPY_ERROR : boto.exception.S3CopyError, + STORAGE_CREATE_ERROR : boto.exception.S3CreateError, + STORAGE_DATA_ERROR : boto.exception.S3DataError, + STORAGE_PERMISSIONS_ERROR : boto.exception.S3PermissionsError, + STORAGE_RESPONSE_ERROR : boto.exception.S3ResponseError, + }, + 'google' : { + STORAGE_COPY_ERROR : boto.exception.GSCopyError, + STORAGE_CREATE_ERROR : boto.exception.GSCreateError, + STORAGE_DATA_ERROR : boto.exception.GSDataError, + STORAGE_PERMISSIONS_ERROR : boto.exception.GSPermissionsError, + STORAGE_RESPONSE_ERROR : boto.exception.GSResponseError, + } + } + + def __init__(self, name, access_key=None, secret_key=None): + self.host = None + self.access_key = access_key + self.secret_key = secret_key + self.name = name + self.acl_class = self.AclClassMap[self.name] + self.canned_acls = self.CannedAclsMap[self.name] + self.get_credentials(access_key, secret_key) + self.configure_headers() + self.configure_errors() + # allow config file to override default host + host_opt_name = '%s_host' % self.HostKeyMap[self.name] + if config.has_option('Credentials', host_opt_name): + self.host = config.get('Credentials', host_opt_name) + + def get_credentials(self, access_key=None, secret_key=None): + access_key_name, secret_key_name = self.CredentialMap[self.name] + if access_key is not None: + self.access_key = access_key + elif os.environ.has_key(access_key_name.upper()): + self.access_key = os.environ[access_key_name.upper()] + elif config.has_option('Credentials', access_key_name): + self.access_key = config.get('Credentials', access_key_name) + + if secret_key is not None: + self.secret_key = secret_key + elif os.environ.has_key(secret_key_name.upper()): + self.secret_key = os.environ[secret_key_name.upper()] + elif config.has_option('Credentials', secret_key_name): + self.secret_key = config.get('Credentials', secret_key_name) + if isinstance(self.secret_key, unicode): + # the secret key must be bytes and not unicode to work + # properly with hmac.new (see http://bugs.python.org/issue5285) + self.secret_key = str(self.secret_key) + + def configure_headers(self): + header_info_map = self.HeaderInfoMap[self.name] + self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY] + self.header_prefix = header_info_map[HEADER_PREFIX_KEY] + self.acl_header = header_info_map[ACL_HEADER_KEY] + self.auth_header = header_info_map[AUTH_HEADER_KEY] + self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY] + self.copy_source_version_id = header_info_map[ + COPY_SOURCE_VERSION_ID_HEADER_KEY] + self.date_header = header_info_map[DATE_HEADER_KEY] + self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY] + self.metadata_directive_header = ( + header_info_map[METADATA_DIRECTIVE_HEADER_KEY]) + self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY] + self.resumable_upload_header = ( + header_info_map[RESUMABLE_UPLOAD_HEADER_KEY]) + self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY] + self.version_id = header_info_map[VERSION_ID_HEADER_KEY] + self.mfa_header = header_info_map[MFA_HEADER_KEY] + + def configure_errors(self): + error_map = self.ErrorMap[self.name] + self.storage_copy_error = error_map[STORAGE_COPY_ERROR] + self.storage_create_error = error_map[STORAGE_CREATE_ERROR] + self.storage_data_error = error_map[STORAGE_DATA_ERROR] + self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR] + self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR] + + def get_provider_name(self): + return self.HostKeyMap[self.name] + +# Static utility method for getting default Provider. +def get_default(): + return Provider('aws') diff --git a/backup/src/boto/pyami/__init__.py b/backup/src/boto/pyami/__init__.py new file mode 100644 index 0000000..303dbb6 --- /dev/null +++ b/backup/src/boto/pyami/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/backup/src/boto/pyami/bootstrap.py b/backup/src/boto/pyami/bootstrap.py new file mode 100644 index 0000000..c1441fd --- /dev/null +++ b/backup/src/boto/pyami/bootstrap.py @@ -0,0 +1,125 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import os +import boto +from boto.utils import get_instance_metadata, get_instance_userdata +from boto.pyami.config import Config, BotoConfigPath +from boto.pyami.scriptbase import ScriptBase + +class Bootstrap(ScriptBase): + """ + The Bootstrap class is instantiated and run as part of the PyAMI + instance initialization process. The methods in this class will + be run from the rc.local script of the instance and will be run + as the root user. + + The main purpose of this class is to make sure the boto distribution + on the instance is the one required. + """ + + def __init__(self): + self.working_dir = '/mnt/pyami' + self.write_metadata() + ScriptBase.__init__(self) + + def write_metadata(self): + fp = open(os.path.expanduser(BotoConfigPath), 'w') + fp.write('[Instance]\n') + inst_data = get_instance_metadata() + for key in inst_data: + fp.write('%s = %s\n' % (key, inst_data[key])) + user_data = get_instance_userdata() + fp.write('\n%s\n' % user_data) + fp.write('[Pyami]\n') + fp.write('working_dir = %s\n' % self.working_dir) + fp.close() + # This file has the AWS credentials, should we lock it down? + # os.chmod(BotoConfigPath, stat.S_IREAD | stat.S_IWRITE) + # now that we have written the file, read it into a pyami Config object + boto.config = Config() + boto.init_logging() + + def create_working_dir(self): + boto.log.info('Working directory: %s' % self.working_dir) + if not os.path.exists(self.working_dir): + os.mkdir(self.working_dir) + + def load_boto(self): + update = boto.config.get('Boto', 'boto_update', 'svn:HEAD') + if update.startswith('svn'): + if update.find(':') >= 0: + method, version = update.split(':') + version = '-r%s' % version + else: + version = '-rHEAD' + location = boto.config.get('Boto', 'boto_location', '/usr/local/boto') + self.run('svn update %s %s' % (version, location)) + elif update.startswith('git'): + location = boto.config.get('Boto', 'boto_location', '/usr/share/python-support/python-boto/boto') + self.run('git pull', cwd=location) + if update.find(':') >= 0: + method, version = update.split(':') + else: + version = 'master' + self.run('git checkout %s' % version, cwd=location) + else: + # first remove the symlink needed when running from subversion + self.run('rm /usr/local/lib/python2.5/site-packages/boto') + self.run('easy_install %s' % update) + + def fetch_s3_file(self, s3_file): + try: + from boto.utils import fetch_file + f = fetch_file(s3_file) + path = os.path.join(self.working_dir, s3_file.split("/")[-1]) + open(path, "w").write(f.read()) + except: + boto.log.exception('Problem Retrieving file: %s' % s3_file) + path = None + return path + + def load_packages(self): + package_str = boto.config.get('Pyami', 'packages') + if package_str: + packages = package_str.split(',') + for package in packages: + package = package.strip() + if package.startswith('s3:'): + package = self.fetch_s3_file(package) + if package: + # if the "package" is really a .py file, it doesn't have to + # be installed, just being in the working dir is enough + if not package.endswith('.py'): + self.run('easy_install -Z %s' % package, exit_on_error=False) + + def main(self): + self.create_working_dir() + self.load_boto() + self.load_packages() + self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id')) + +if __name__ == "__main__": + # because bootstrap starts before any logging configuration can be loaded from + # the boto config files, we will manually enable logging to /var/log/boto.log + boto.set_file_logger('bootstrap', '/var/log/boto.log') + bs = Bootstrap() + bs.main() diff --git a/backup/src/boto/pyami/config.py b/backup/src/boto/pyami/config.py new file mode 100644 index 0000000..f4613ab --- /dev/null +++ b/backup/src/boto/pyami/config.py @@ -0,0 +1,203 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import StringIO, os, re +import ConfigParser +import boto + +BotoConfigPath = '/etc/boto.cfg' +BotoConfigLocations = [BotoConfigPath] +if 'BOTO_CONFIG' in os.environ: + BotoConfigLocations = [os.path.expanduser(os.environ['BOTO_CONFIG'])] +elif 'HOME' in os.environ: + UserConfigPath = os.path.expanduser('~/.boto') + BotoConfigLocations.append(UserConfigPath) +else: + UserConfigPath = None + +class Config(ConfigParser.SafeConfigParser): + + def __init__(self, path=None, fp=None, do_load=True): + ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami', + 'debug' : '0'}) + if do_load: + if path: + self.load_from_path(path) + elif fp: + self.readfp(fp) + else: + self.read(BotoConfigLocations) + if "AWS_CREDENTIAL_FILE" in os.environ: + self.load_credential_file(os.path.expanduser(os.environ['AWS_CREDENTIAL_FILE'])) + + def load_credential_file(self, path): + """Load a credential file as is setup like the Java utilities""" + c_data = StringIO.StringIO() + c_data.write("[Credentials]\n") + for line in open(path, "r").readlines(): + c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key")) + c_data.seek(0) + self.readfp(c_data) + + def load_from_path(self, path): + file = open(path) + for line in file.readlines(): + match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line) + if match: + extended_file = match.group(1) + (dir, file) = os.path.split(path) + self.load_from_path(os.path.join(dir, extended_file)) + self.read(path) + + def save_option(self, path, section, option, value): + """ + Write the specified Section.Option to the config file specified by path. + Replace any previous value. If the path doesn't exist, create it. + Also add the option the the in-memory config. + """ + config = ConfigParser.SafeConfigParser() + config.read(path) + if not config.has_section(section): + config.add_section(section) + config.set(section, option, value) + fp = open(path, 'w') + config.write(fp) + fp.close() + if not self.has_section(section): + self.add_section(section) + self.set(section, option, value) + + def save_user_option(self, section, option, value): + self.save_option(UserConfigPath, section, option, value) + + def save_system_option(self, section, option, value): + self.save_option(BotoConfigPath, section, option, value) + + def get_instance(self, name, default=None): + try: + val = self.get('Instance', name) + except: + val = default + return val + + def get_user(self, name, default=None): + try: + val = self.get('User', name) + except: + val = default + return val + + def getint_user(self, name, default=0): + try: + val = self.getint('User', name) + except: + val = default + return val + + def get_value(self, section, name, default=None): + return self.get(section, name, default) + + def get(self, section, name, default=None): + try: + val = ConfigParser.SafeConfigParser.get(self, section, name) + except: + val = default + return val + + def getint(self, section, name, default=0): + try: + val = ConfigParser.SafeConfigParser.getint(self, section, name) + except: + val = int(default) + return val + + def getfloat(self, section, name, default=0.0): + try: + val = ConfigParser.SafeConfigParser.getfloat(self, section, name) + except: + val = float(default) + return val + + def getbool(self, section, name, default=False): + if self.has_option(section, name): + val = self.get(section, name) + if val.lower() == 'true': + val = True + else: + val = False + else: + val = default + return val + + def setbool(self, section, name, value): + if value: + self.set(section, name, 'true') + else: + self.set(section, name, 'false') + + def dump(self): + s = StringIO.StringIO() + self.write(s) + print s.getvalue() + + def dump_safe(self, fp=None): + if not fp: + fp = StringIO.StringIO() + for section in self.sections(): + fp.write('[%s]\n' % section) + for option in self.options(section): + if option == 'aws_secret_access_key': + fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option) + else: + fp.write('%s = %s\n' % (option, self.get(section, option))) + + def dump_to_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + if not domain: + domain = sdb.create_domain(domain_name) + item = domain.new_item(item_name) + item.active = False + for section in self.sections(): + d = {} + for option in self.options(section): + d[option] = self.get(section, option) + item[section] = simplejson.dumps(d) + item.save() + + def load_from_sdb(self, domain_name, item_name): + import simplejson + sdb = boto.connect_sdb() + domain = sdb.lookup(domain_name) + item = domain.get_item(item_name) + for section in item.keys(): + if not self.has_section(section): + self.add_section(section) + d = simplejson.loads(item[section]) + for attr_name in d.keys(): + attr_value = d[attr_name] + if attr_value == None: + attr_value = 'None' + if isinstance(attr_value, bool): + self.setbool(section, attr_name, attr_value) + else: + self.set(section, attr_name, attr_value) diff --git a/backup/src/boto/pyami/copybot.cfg b/backup/src/boto/pyami/copybot.cfg new file mode 100644 index 0000000..cbfdc5a --- /dev/null +++ b/backup/src/boto/pyami/copybot.cfg @@ -0,0 +1,60 @@ +# +# Your AWS Credentials +# +[Credentials] +aws_access_key_id = +aws_secret_access_key = + +# +# If you want to use a separate set of credentials when writing +# to the destination bucket, put them here +#dest_aws_access_key_id = +#dest_aws_secret_access_key = + +# +# Fill out this section if you want emails from CopyBot +# when it starts and stops +# +[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +# +# If you leave this section as is, it will automatically +# update boto from subversion upon start up. +# If you don't want that to happen, comment this out +# +[Boto] +boto_location = /usr/local/boto +boto_update = svn:HEAD + +# +# This tells the Pyami code in boto what scripts +# to run during startup +# +[Pyami] +scripts = boto.pyami.copybot.CopyBot + +# +# Source bucket and Destination Bucket, obviously. +# If the Destination bucket does not exist, it will +# attempt to create it. +# If exit_on_completion is false, the instance +# will keep running after the copy operation is +# complete which might be handy for debugging. +# If copy_acls is false, the ACL's will not be +# copied with the objects to the new bucket. +# If replace_dst is false, copybot will not +# will only store the source file in the dest if +# that file does not already exist. If it's true +# it will replace it even if it does exist. +# +[CopyBot] +src_bucket = +dst_bucket = +exit_on_completion = true +copy_acls = true +replace_dst = true diff --git a/backup/src/boto/pyami/copybot.py b/backup/src/boto/pyami/copybot.py new file mode 100644 index 0000000..ed397cb --- /dev/null +++ b/backup/src/boto/pyami/copybot.py @@ -0,0 +1,97 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto +from boto.pyami.scriptbase import ScriptBase +import os, StringIO + +class CopyBot(ScriptBase): + + def __init__(self): + ScriptBase.__init__(self) + self.wdir = boto.config.get('Pyami', 'working_dir') + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.wdir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + self.src_name = boto.config.get(self.name, 'src_bucket') + self.dst_name = boto.config.get(self.name, 'dst_bucket') + self.replace = boto.config.getbool(self.name, 'replace_dst', True) + s3 = boto.connect_s3() + self.src = s3.lookup(self.src_name) + if not self.src: + boto.log.error('Source bucket does not exist: %s' % self.src_name) + dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None) + if dest_access_key: + dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None) + s3 = boto.connect(dest_access_key, dest_secret_key) + self.dst = s3.lookup(self.dst_name) + if not self.dst: + self.dst = s3.create_bucket(self.dst_name) + + def copy_bucket_acl(self): + if boto.config.get(self.name, 'copy_acls', True): + acl = self.src.get_xml_acl() + self.dst.set_xml_acl(acl) + + def copy_key_acl(self, src, dst): + if boto.config.get(self.name, 'copy_acls', True): + acl = src.get_xml_acl() + dst.set_xml_acl(acl) + + def copy_keys(self): + boto.log.info('src=%s' % self.src.name) + boto.log.info('dst=%s' % self.dst.name) + try: + for key in self.src: + if not self.replace: + exists = self.dst.lookup(key.name) + if exists: + boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name)) + continue + boto.log.info('copying %d bytes from key=%s' % (key.size, key.name)) + prefix, base = os.path.split(key.name) + path = os.path.join(self.wdir, base) + key.get_contents_to_filename(path) + new_key = self.dst.new_key(key.name) + new_key.set_contents_from_filename(path) + self.copy_key_acl(key, new_key) + os.unlink(path) + except: + boto.log.exception('Error copying key: %s' % key.name) + + def copy_log(self): + key = self.dst.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + + def main(self): + fp = StringIO.StringIO() + boto.config.dump_safe(fp) + self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue()) + if self.src and self.dst: + self.copy_keys() + if self.dst: + self.copy_log() + self.notify('%s (%s) Stopping' % (self.name, self.instance_id), + 'Copy Operation Complete') + if boto.config.getbool(self.name, 'exit_on_completion', True): + ec2 = boto.connect_ec2() + ec2.terminate_instances([self.instance_id]) + diff --git a/backup/src/boto/pyami/helloworld.py b/backup/src/boto/pyami/helloworld.py new file mode 100644 index 0000000..680873c --- /dev/null +++ b/backup/src/boto/pyami/helloworld.py @@ -0,0 +1,28 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase + +class HelloWorld(ScriptBase): + + def main(self): + self.log('Hello World!!!') + diff --git a/backup/src/boto/pyami/installers/__init__.py b/backup/src/boto/pyami/installers/__init__.py new file mode 100644 index 0000000..cc68926 --- /dev/null +++ b/backup/src/boto/pyami/installers/__init__.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.scriptbase import ScriptBase + + +class Installer(ScriptBase): + """ + Abstract base class for installers + """ + + def add_cron(self, name, minute, hour, mday, month, wday, who, command, env=None): + """ + Add an entry to the system crontab. + """ + raise NotImplementedError + + def add_init_script(self, file): + """ + Add this file to the init.d directory + """ + + def add_env(self, key, value): + """ + Add an environemnt variable + """ + raise NotImplementedError + + def stop(self, service_name): + """ + Stop a service. + """ + raise NotImplementedError + + def start(self, service_name): + """ + Start a service. + """ + raise NotImplementedError + + def install(self): + """ + Do whatever is necessary to "install" the package. + """ + raise NotImplementedError + diff --git a/backup/src/boto/pyami/installers/ubuntu/__init__.py b/backup/src/boto/pyami/installers/ubuntu/__init__.py new file mode 100644 index 0000000..60ee658 --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/__init__.py @@ -0,0 +1,22 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/backup/src/boto/pyami/installers/ubuntu/apache.py b/backup/src/boto/pyami/installers/ubuntu/apache.py new file mode 100644 index 0000000..febc2df --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/apache.py @@ -0,0 +1,43 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer + +class Apache(Installer): + """ + Install apache2, mod_python, and libapache2-svn + """ + + def install(self): + self.run("apt-get update") + self.run('apt-get -y install apache2', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-mod-python', notify=True, exit_on_error=True) + self.run('a2enmod rewrite', notify=True, exit_on_error=True) + self.run('a2enmod ssl', notify=True, exit_on_error=True) + self.run('a2enmod proxy', notify=True, exit_on_error=True) + self.run('a2enmod proxy_ajp', notify=True, exit_on_error=True) + + # Hard reboot the apache2 server to enable these module + self.stop("apache2") + self.start("apache2") + + def main(self): + self.install() diff --git a/backup/src/boto/pyami/installers/ubuntu/ebs.py b/backup/src/boto/pyami/installers/ubuntu/ebs.py new file mode 100644 index 0000000..204c9b1 --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/ebs.py @@ -0,0 +1,220 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +Automated installer to attach, format and mount an EBS volume. +This installer assumes that you want the volume formatted as +an XFS file system. To drive this installer, you need the +following section in the boto config passed to the new instance. +You also need to install dateutil by listing python-dateutil +in the list of packages to be installed in the Pyami seciont +of your boto config file. + +If there is already a device mounted at the specified mount point, +the installer assumes that it is the ephemeral drive and unmounts +it, remounts it as /tmp and chmods it to 777. + +Config file section:: + + [EBS] + volume_id = + logical_volume_name = + device = + mount_point = + +""" +import boto +from boto.manage.volume import Volume +import os, time +from boto.pyami.installers.ubuntu.installer import Installer +from string import Template + +BackupScriptTemplate = """#!/usr/bin/env python +# Backup EBS volume +import boto +from boto.pyami.scriptbase import ScriptBase +import traceback + +class Backup(ScriptBase): + + def main(self): + try: + ec2 = boto.connect_ec2() + self.run("/usr/sbin/xfs_freeze -f ${mount_point}") + snapshot = ec2.create_snapshot('${volume_id}') + boto.log.info("Snapshot created: %s " % snapshot) + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + boto.log.info("Snapshot created: ${volume_id}") + except Exception, e: + self.notify(subject="${instance_id} Backup Failed", body=traceback.format_exc()) + finally: + self.run("/usr/sbin/xfs_freeze -u ${mount_point}") + +if __name__ == "__main__": + b = Backup() + b.main() +""" + +BackupCleanupScript= """#!/usr/bin/env python +import boto +from boto.manage.volume import Volume + +# Cleans Backups of EBS volumes + +for v in Volume.all(): + v.trim_snapshots(True) +""" + +class EBSInstaller(Installer): + """ + Set up the EBS stuff + """ + + def __init__(self, config_file=None): + Installer.__init__(self, config_file) + self.instance_id = boto.config.get('Instance', 'instance-id') + self.device = boto.config.get('EBS', 'device', '/dev/sdp') + self.volume_id = boto.config.get('EBS', 'volume_id') + self.logical_volume_name = boto.config.get('EBS', 'logical_volume_name') + self.mount_point = boto.config.get('EBS', 'mount_point', '/ebs') + + def attach(self): + ec2 = boto.connect_ec2() + if self.logical_volume_name: + # if a logical volume was specified, override the specified volume_id + # (if there was one) with the current AWS volume for the logical volume: + logical_volume = Volume.find(name = self.logical_volume_name).next() + self.volume_id = logical_volume._volume_id + volume = ec2.get_all_volumes([self.volume_id])[0] + # wait for the volume to be available. The volume may still be being created + # from a snapshot. + while volume.update() != 'available': + boto.log.info('Volume %s not yet available. Current status = %s.' % (volume.id, volume.status)) + time.sleep(5) + instance = ec2.get_all_instances([self.instance_id])[0].instances[0] + attempt_attach = True + while attempt_attach: + try: + ec2.attach_volume(self.volume_id, self.instance_id, self.device) + attempt_attach = False + except EC2ResponseError, e: + if e.error_code != 'IncorrectState': + # if there's an EC2ResonseError with the code set to IncorrectState, delay a bit for ec2 + # to realize the instance is running, then try again. Otherwise, raise the error: + boto.log.info('Attempt to attach the EBS volume %s to this instance (%s) returned %s. Trying again in a bit.' % (self.volume_id, self.instance_id, e.errors)) + time.sleep(2) + else: + raise e + boto.log.info('Attached volume %s to instance %s as device %s' % (self.volume_id, self.instance_id, self.device)) + # now wait for the volume device to appear + while not os.path.exists(self.device): + boto.log.info('%s still does not exist, waiting 2 seconds' % self.device) + time.sleep(2) + + def make_fs(self): + boto.log.info('make_fs...') + has_fs = self.run('fsck %s' % self.device) + if has_fs != 0: + self.run('mkfs -t xfs %s' % self.device) + + def create_backup_script(self): + t = Template(BackupScriptTemplate) + s = t.substitute(volume_id=self.volume_id, instance_id=self.instance_id, + mount_point=self.mount_point) + fp = open('/usr/local/bin/ebs_backup', 'w') + fp.write(s) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup') + + def create_backup_cleanup_script(self): + fp = open('/usr/local/bin/ebs_backup_cleanup', 'w') + fp.write(BackupCleanupScript) + fp.close() + self.run('chmod +x /usr/local/bin/ebs_backup_cleanup') + + def handle_mount_point(self): + boto.log.info('handle_mount_point') + if not os.path.isdir(self.mount_point): + boto.log.info('making directory') + # mount directory doesn't exist so create it + self.run("mkdir %s" % self.mount_point) + else: + boto.log.info('directory exists already') + self.run('mount -l') + lines = self.last_command.output.split('\n') + for line in lines: + t = line.split() + if t and t[2] == self.mount_point: + # something is already mounted at the mount point + # unmount that and mount it as /tmp + if t[0] != self.device: + self.run('umount %s' % self.mount_point) + self.run('mount %s /tmp' % t[0]) + break + self.run('chmod 777 /tmp') + # Mount up our new EBS volume onto mount_point + self.run("mount %s %s" % (self.device, self.mount_point)) + self.run('xfs_growfs %s' % self.mount_point) + + def update_fstab(self): + f = open("/etc/fstab", "a") + f.write('%s\t%s\txfs\tdefaults 0 0\n' % (self.device, self.mount_point)) + f.close() + + def install(self): + # First, find and attach the volume + self.attach() + + # Install the xfs tools + self.run('apt-get -y install xfsprogs xfsdump') + + # Check to see if the filesystem was created or not + self.make_fs() + + # create the /ebs directory for mounting + self.handle_mount_point() + + # create the backup script + self.create_backup_script() + + # Set up the backup script + minute = boto.config.get('EBS', 'backup_cron_minute', '0') + hour = boto.config.get('EBS', 'backup_cron_hour', '4,16') + self.add_cron("ebs_backup", "/usr/local/bin/ebs_backup", minute=minute, hour=hour) + + # Set up the backup cleanup script + minute = boto.config.get('EBS', 'backup_cleanup_cron_minute') + hour = boto.config.get('EBS', 'backup_cleanup_cron_hour') + if (minute != None) and (hour != None): + self.create_backup_cleanup_script(); + self.add_cron("ebs_backup_cleanup", "/usr/local/bin/ebs_backup_cleanup", minute=minute, hour=hour) + + # Set up the fstab + self.update_fstab() + + def main(self): + if not os.path.exists(self.device): + self.install() + else: + boto.log.info("Device %s is already attached, skipping EBS Installer" % self.device) diff --git a/backup/src/boto/pyami/installers/ubuntu/installer.py b/backup/src/boto/pyami/installers/ubuntu/installer.py new file mode 100644 index 0000000..370d63f --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/installer.py @@ -0,0 +1,96 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import boto.pyami.installers +import os +import os.path +import stat +import boto +import random +from pwd import getpwnam + +class Installer(boto.pyami.installers.Installer): + """ + Base Installer class for Ubuntu-based AMI's + """ + def add_cron(self, name, command, minute="*", hour="*", mday="*", month="*", wday="*", who="root", env=None): + """ + Write a file to /etc/cron.d to schedule a command + env is a dict containing environment variables you want to set in the file + name will be used as the name of the file + """ + if minute == 'random': + minute = str(random.randrange(60)) + if hour == 'random': + hour = str(random.randrange(24)) + fp = open('/etc/cron.d/%s' % name, "w") + if env: + for key, value in env.items(): + fp.write('%s=%s\n' % (key, value)) + fp.write('%s %s %s %s %s %s %s\n' % (minute, hour, mday, month, wday, who, command)) + fp.close() + + def add_init_script(self, file, name): + """ + Add this file to the init.d directory + """ + f_path = os.path.join("/etc/init.d", name) + f = open(f_path, "w") + f.write(file) + f.close() + os.chmod(f_path, stat.S_IREAD| stat.S_IWRITE | stat.S_IEXEC) + self.run("/usr/sbin/update-rc.d %s defaults" % name) + + def add_env(self, key, value): + """ + Add an environemnt variable + For Ubuntu, the best place is /etc/environment. Values placed here do + not need to be exported. + """ + boto.log.info('Adding env variable: %s=%s' % (key, value)) + if not os.path.exists("/etc/environment.orig"): + self.run('cp /etc/environment /etc/environment.orig', notify=False, exit_on_error=False) + fp = open('/etc/environment', 'a') + fp.write('\n%s="%s"' % (key, value)) + fp.close() + os.environ[key] = value + + def stop(self, service_name): + self.run('/etc/init.d/%s stop' % service_name) + + def start(self, service_name): + self.run('/etc/init.d/%s start' % service_name) + + def create_user(self, user): + """ + Create a user on the local system + """ + self.run("useradd -m %s" % user) + usr = getpwnam(user) + return usr + + + def install(self): + """ + This is the only method you need to override + """ + raise NotImplementedError + diff --git a/backup/src/boto/pyami/installers/ubuntu/mysql.py b/backup/src/boto/pyami/installers/ubuntu/mysql.py new file mode 100644 index 0000000..490e5db --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/mysql.py @@ -0,0 +1,109 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +""" +This installer will install mysql-server on an Ubuntu machine. +In addition to the normal installation done by apt-get, it will +also configure the new MySQL server to store it's data files in +a different location. By default, this is /mnt but that can be +configured in the [MySQL] section of the boto config file passed +to the instance. +""" +from boto.pyami.installers.ubuntu.installer import Installer +import os +import boto +from boto.utils import ShellCommand +from ConfigParser import SafeConfigParser +import time + +ConfigSection = """ +[MySQL] +root_password = +data_dir = +""" + +class MySQL(Installer): + + def install(self): + self.run('apt-get update') + self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True) + +# def set_root_password(self, password=None): +# if not password: +# password = boto.config.get('MySQL', 'root_password') +# if password: +# self.run('mysqladmin -u root password %s' % password) +# return password + + def change_data_dir(self, password=None): + data_dir = boto.config.get('MySQL', 'data_dir', '/mnt') + fresh_install = False; + is_mysql_running_command = ShellCommand('mysqladmin ping') # exit status 0 if mysql is running + is_mysql_running_command.run() + if is_mysql_running_command.getStatus() == 0: + # mysql is running. This is the state apt-get will leave it in. If it isn't running, + # that means mysql was already installed on the AMI and there's no need to stop it, + # saving 40 seconds on instance startup. + time.sleep(10) #trying to stop mysql immediately after installing it fails + # We need to wait until mysql creates the root account before we kill it + # or bad things will happen + i = 0 + while self.run("echo 'quit' | mysql -u root") != 0 and i<5: + time.sleep(5) + i = i + 1 + self.run('/etc/init.d/mysql stop') + self.run("pkill -9 mysql") + + mysql_path = os.path.join(data_dir, 'mysql') + if not os.path.exists(mysql_path): + self.run('mkdir %s' % mysql_path) + fresh_install = True; + self.run('chown -R mysql:mysql %s' % mysql_path) + fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w') + fp.write('# created by pyami\n') + fp.write('# use the %s volume for data\n' % data_dir) + fp.write('[mysqld]\n') + fp.write('datadir = %s\n' % mysql_path) + fp.write('log_bin = %s\n' % os.path.join(mysql_path, 'mysql-bin.log')) + fp.close() + if fresh_install: + self.run('cp -pr /var/lib/mysql/* %s/' % mysql_path) + self.start('mysql') + else: + #get the password ubuntu expects to use: + config_parser = SafeConfigParser() + config_parser.read('/etc/mysql/debian.cnf') + password = config_parser.get('client', 'password') + # start the mysql deamon, then mysql with the required grant statement piped into it: + self.start('mysql') + time.sleep(10) #time for mysql to start + grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password + while self.run(grant_command) != 0: + time.sleep(5) + # leave mysqld running + + def main(self): + self.install() + # change_data_dir runs 'mysql -u root' which assumes there is no mysql password, i + # and changing that is too ugly to be worth it: + #self.set_root_password() + self.change_data_dir() + diff --git a/backup/src/boto/pyami/installers/ubuntu/trac.py b/backup/src/boto/pyami/installers/ubuntu/trac.py new file mode 100644 index 0000000..ef83af7 --- /dev/null +++ b/backup/src/boto/pyami/installers/ubuntu/trac.py @@ -0,0 +1,139 @@ +# Copyright (c) 2008 Chris Moyer http://coredumped.org +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +from boto.pyami.installers.ubuntu.installer import Installer +import boto +import os + +class Trac(Installer): + """ + Install Trac and DAV-SVN + Sets up a Vhost pointing to [Trac]->home + Using the config parameter [Trac]->hostname + Sets up a trac environment for every directory found under [Trac]->data_dir + + [Trac] + name = My Foo Server + hostname = trac.foo.com + home = /mnt/sites/trac + data_dir = /mnt/trac + svn_dir = /mnt/subversion + server_admin = root@foo.com + sdb_auth_domain = users + # Optional + SSLCertificateFile = /mnt/ssl/foo.crt + SSLCertificateKeyFile = /mnt/ssl/foo.key + SSLCertificateChainFile = /mnt/ssl/FooCA.crt + + """ + + def install(self): + self.run('apt-get -y install trac', notify=True, exit_on_error=True) + self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True) + self.run("a2enmod ssl") + self.run("a2enmod mod_python") + self.run("a2enmod dav_svn") + self.run("a2enmod rewrite") + # Make sure that boto.log is writable by everyone so that subversion post-commit hooks can + # write to it. + self.run("touch /var/log/boto.log") + self.run("chmod a+w /var/log/boto.log") + + def setup_vhost(self): + domain = boto.config.get("Trac", "hostname").strip() + if domain: + domain_info = domain.split('.') + cnf = open("/etc/apache2/sites-available/%s" % domain_info[0], "w") + cnf.write("NameVirtualHost *:80\n") + if boto.config.get("Trac", "SSLCertificateFile"): + cnf.write("NameVirtualHost *:443\n\n") + cnf.write("\n") + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tRewriteEngine On\n") + cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain) + cnf.write("\n\n") + + cnf.write("\n") + else: + cnf.write("\n") + + cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip()) + cnf.write("\tServerName %s\n" % domain) + cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip()) + + cnf.write("\t\n" % boto.config.get("Trac", "home").strip()) + cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n") + cnf.write("\t\tAllowOverride All\n") + cnf.write("\t\tOrder allow,deny\n") + cnf.write("\t\tallow from all\n") + cnf.write("\t\n") + + cnf.write("\t\n") + cnf.write("\t\tAuthType Basic\n") + cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name")) + cnf.write("\t\tRequire valid-user\n") + cnf.write("\t\tAuthUserFile /mnt/apache/passwd/passwords\n") + cnf.write("\t\n") + + data_dir = boto.config.get("Trac", "data_dir") + for env in os.listdir(data_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tSetHandler mod_python\n") + cnf.write("\t\tPythonInterpreter main_interpreter\n") + cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n") + cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env)) + cnf.write("\t\tPythonOption TracUriRoot /trac/%s\n" % env) + cnf.write("\t\n") + + svn_dir = boto.config.get("Trac", "svn_dir") + for env in os.listdir(svn_dir): + if(env[0] != "."): + cnf.write("\t\n" % env) + cnf.write("\t\tDAV svn\n") + cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env)) + cnf.write("\t\n") + + cnf.write("\tErrorLog /var/log/apache2/error.log\n") + cnf.write("\tLogLevel warn\n") + cnf.write("\tCustomLog /var/log/apache2/access.log combined\n") + cnf.write("\tServerSignature On\n") + SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile") + if SSLCertificateFile: + cnf.write("\tSSLEngine On\n") + cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile) + + SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile") + if SSLCertificateKeyFile: + cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile) + + SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile") + if SSLCertificateChainFile: + cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile) + cnf.write("\n") + cnf.close() + self.run("a2ensite %s" % domain_info[0]) + self.run("/etc/init.d/apache2 force-reload") + + def main(self): + self.install() + self.setup_vhost() diff --git a/backup/src/boto/pyami/launch_ami.py b/backup/src/boto/pyami/launch_ami.py new file mode 100644 index 0000000..243d56d --- /dev/null +++ b/backup/src/boto/pyami/launch_ami.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import getopt +import sys +import imp +import time +import boto + +usage_string = """ +SYNOPSIS + launch_ami.py -a ami_id [-b script_bucket] [-s script_name] + [-m module] [-c class_name] [-r] + [-g group] [-k key_name] [-n num_instances] + [-w] [extra_data] + Where: + ami_id - the id of the AMI you wish to launch + module - The name of the Python module containing the class you + want to run when the instance is started. If you use this + option the Python module must already be stored on the + instance in a location that is on the Python path. + script_file - The name of a local Python module that you would like + to have copied to S3 and then run on the instance + when it is started. The specified module must be + import'able (i.e. in your local Python path). It + will then be copied to the specified bucket in S3 + (see the -b option). Once the new instance(s) + start up the script will be copied from S3 and then + run locally on the instance. + class_name - The name of the class to be instantiated within the + module or script file specified. + script_bucket - the name of the bucket in which the script will be + stored + group - the name of the security group the instance will run in + key_name - the name of the keypair to use when launching the AMI + num_instances - how many instances of the AMI to launch (default 1) + input_queue_name - Name of SQS to read input messages from + output_queue_name - Name of SQS to write output messages to + extra_data - additional name-value pairs that will be passed as + userdata to the newly launched instance. These should + be of the form "name=value" + The -r option reloads the Python module to S3 without launching + another instance. This can be useful during debugging to allow + you to test a new version of your script without shutting down + your instance and starting up another one. + The -w option tells the script to run synchronously, meaning to + wait until the instance is actually up and running. It then prints + the IP address and internal and external DNS names before exiting. +""" + +def usage(): + print usage_string + sys.exit() + +def main(): + try: + opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w', + ['ami', 'bucket', 'class', 'group', 'help', + 'inputqueue', 'keypair', 'module', + 'numinstances', 'outputqueue', + 'reload', 'script_name', 'wait']) + except: + usage() + params = {'module_name' : None, + 'script_name' : None, + 'class_name' : None, + 'script_bucket' : None, + 'group' : 'default', + 'keypair' : None, + 'ami' : None, + 'num_instances' : 1, + 'input_queue_name' : None, + 'output_queue_name' : None} + reload = None + wait = None + for o, a in opts: + if o in ('-a', '--ami'): + params['ami'] = a + if o in ('-b', '--bucket'): + params['script_bucket'] = a + if o in ('-c', '--class'): + params['class_name'] = a + if o in ('-g', '--group'): + params['group'] = a + if o in ('-h', '--help'): + usage() + if o in ('-i', '--inputqueue'): + params['input_queue_name'] = a + if o in ('-k', '--keypair'): + params['keypair'] = a + if o in ('-m', '--module'): + params['module_name'] = a + if o in ('-n', '--num_instances'): + params['num_instances'] = int(a) + if o in ('-o', '--outputqueue'): + params['output_queue_name'] = a + if o in ('-r', '--reload'): + reload = True + if o in ('-s', '--script'): + params['script_name'] = a + if o in ('-w', '--wait'): + wait = True + + # check required fields + required = ['ami'] + for pname in required: + if not params.get(pname, None): + print '%s is required' % pname + usage() + if params['script_name']: + # first copy the desired module file to S3 bucket + if reload: + print 'Reloading module %s to S3' % params['script_name'] + else: + print 'Copying module %s to S3' % params['script_name'] + l = imp.find_module(params['script_name']) + c = boto.connect_s3() + bucket = c.get_bucket(params['script_bucket']) + key = bucket.new_key(params['script_name']+'.py') + key.set_contents_from_file(l[0]) + params['script_md5'] = key.md5 + # we have everything we need, now build userdata string + l = [] + for k, v in params.items(): + if v: + l.append('%s=%s' % (k, v)) + c = boto.connect_ec2() + l.append('aws_access_key_id=%s' % c.aws_access_key_id) + l.append('aws_secret_access_key=%s' % c.aws_secret_access_key) + for kv in args: + l.append(kv) + s = '|'.join(l) + if not reload: + rs = c.get_all_images([params['ami']]) + img = rs[0] + r = img.run(user_data=s, key_name=params['keypair'], + security_groups=[params['group']], + max_count=params.get('num_instances', 1)) + print 'AMI: %s - %s (Started)' % (params['ami'], img.location) + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + if wait: + running = False + while not running: + time.sleep(30) + [i.update() for i in r.instances] + status = [i.state for i in r.instances] + print status + if status.count('running') == len(r.instances): + running = True + for i in r.instances: + print 'Instance: %s' % i.ami_launch_index + print 'Public DNS Name: %s' % i.public_dns_name + print 'Private DNS Name: %s' % i.private_dns_name + +if __name__ == "__main__": + main() + diff --git a/backup/src/boto/pyami/scriptbase.py b/backup/src/boto/pyami/scriptbase.py new file mode 100644 index 0000000..90522ca --- /dev/null +++ b/backup/src/boto/pyami/scriptbase.py @@ -0,0 +1,44 @@ +import os +import sys +from boto.utils import ShellCommand, get_ts +import boto +import boto.utils + +class ScriptBase: + + def __init__(self, config_file=None): + self.instance_id = boto.config.get('Instance', 'instance-id', 'default') + self.name = self.__class__.__name__ + self.ts = get_ts() + if config_file: + boto.config.read(config_file) + + def notify(self, subject, body=''): + boto.utils.notify(subject, body) + + def mkdir(self, path): + if not os.path.isdir(path): + try: + os.mkdir(path) + except: + boto.log.error('Error creating directory: %s' % path) + + def umount(self, path): + if os.path.ismount(path): + self.run('umount %s' % path) + + def run(self, command, notify=True, exit_on_error=False, cwd=None): + self.last_command = ShellCommand(command, cwd=cwd) + if self.last_command.status != 0: + boto.log.error('Error running command: "%s". Output: "%s"' % (command, self.last_command.output)) + if notify: + self.notify('Error encountered', \ + 'Error running the following command:\n\t%s\n\nCommand output:\n\t%s' % \ + (command, self.last_command.output)) + if exit_on_error: + sys.exit(-1) + return self.last_command.status + + def main(self): + pass + diff --git a/backup/src/boto/pyami/startup.py b/backup/src/boto/pyami/startup.py new file mode 100644 index 0000000..2093151 --- /dev/null +++ b/backup/src/boto/pyami/startup.py @@ -0,0 +1,60 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +import sys +import boto +from boto.utils import find_class +from boto import config +from boto.pyami.scriptbase import ScriptBase + + +class Startup(ScriptBase): + + def run_scripts(self): + scripts = config.get('Pyami', 'scripts') + if scripts: + for script in scripts.split(','): + script = script.strip(" ") + try: + pos = script.rfind('.') + if pos > 0: + mod_name = script[0:pos] + cls_name = script[pos+1:] + cls = find_class(mod_name, cls_name) + boto.log.info('Running Script: %s' % script) + s = cls() + s.main() + else: + boto.log.warning('Trouble parsing script: %s' % script) + except Exception, e: + boto.log.exception('Problem Running Script: %s. Startup process halting.' % script) + raise e + + def main(self): + self.run_scripts() + self.notify('Startup Completed for %s' % config.get('Instance', 'instance-id')) + +if __name__ == "__main__": + if not config.has_section('loggers'): + boto.set_file_logger('startup', '/var/log/boto.log') + sys.path.append(config.get('Pyami', 'working_dir')) + su = Startup() + su.main() diff --git a/backup/src/boto/rds/__init__.py b/backup/src/boto/rds/__init__.py new file mode 100644 index 0000000..940815d --- /dev/null +++ b/backup/src/boto/rds/__init__.py @@ -0,0 +1,972 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import boto.utils +import urllib +from boto.connection import AWSQueryConnection +from boto.rds.dbinstance import DBInstance +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup +from boto.rds.dbsnapshot import DBSnapshot +from boto.rds.event import Event +from boto.rds.regioninfo import RDSRegionInfo + +def regions(): + """ + Get all available regions for the RDS service. + + :rtype: list + :return: A list of :class:`boto.rds.regioninfo.RDSRegionInfo` + """ + return [RDSRegionInfo(name='us-east-1', + endpoint='rds.amazonaws.com'), + RDSRegionInfo(name='eu-west-1', + endpoint='eu-west-1.rds.amazonaws.com'), + RDSRegionInfo(name='us-west-1', + endpoint='us-west-1.rds.amazonaws.com'), + RDSRegionInfo(name='ap-southeast-1', + endpoint='ap-southeast-1.rds.amazonaws.com') + ] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None + +#boto.set_stream_logger('rds') + +class RDSConnection(AWSQueryConnection): + + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'rds.amazonaws.com' + APIVersion = '2009-10-16' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + if not region: + region = RDSRegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, + proxy_pass, self.region.endpoint, debug, + https_connection_factory, path) + + def _required_auth_capability(self): + return ['rds'] + + # DB Instance methods + + def get_all_dbinstances(self, instance_id=None, max_records=None, + marker=None): + """ + Retrieve all the DBInstances in your account. + + :type instance_id: str + :param instance_id: DB Instance identifier. If supplied, only information + this instance will be returned. Otherwise, info + about all DB Instances will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbinstance.DBInstance` + """ + params = {} + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBInstances', params, [('DBInstance', DBInstance)]) + + def create_dbinstance(self, id, allocated_storage, instance_class, + master_username, master_password, port=3306, + engine='MySQL5.1', db_name=None, param_group=None, + security_groups=None, availability_zone=None, + preferred_maintenance_window=None, + backup_retention_period=None, + preferred_backup_window=None, + multi_az=False, + engine_version=None, + auto_minor_version_upgrade=True): + """ + Create a new DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + Must contain 1-63 alphanumeric characters. + First character must be a letter. + May not end with a hyphen or contain two consecutive hyphens + + :type allocated_storage: int + :param allocated_storage: Initially allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + + Valid values are: + + * db.m1.small + * db.m1.large + * db.m1.xlarge + * db.m2.xlarge + * db.m2.2xlarge + * db.m2.4xlarge + + :type engine: str + :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type master_username: str + :param master_username: Name of master user for the DBInstance. + Must be 1-15 alphanumeric characters, first + must be a letter. + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-16 alphanumeric characters. + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type db_name: str + :param db_name: Name of a database to create when the DBInstance + is created. Default is to create no databases. + + :type param_group: str + :param param_group: Name of DBParameterGroup to associate with + this DBInstance. If no groups are specified + no parameter groups will be used. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) + during which maintenance can occur. + Default is Sun:05:00-Sun:09:00 + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :type multi_az: bool + :param multi_az: If True, specifies the DB Instance will be + deployed in multiple availability zones. + + :type engine_version: str + :param engine_version: Version number of the database engine to use. + + :type auto_minor_version_upgrade: bool + :param auto_minor_version_upgrade: Indicates that minor engine + upgrades will be applied + automatically to the Read Replica + during the maintenance window. + Default is True. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The new db instance. + """ + params = {'DBInstanceIdentifier' : id, + 'AllocatedStorage' : allocated_storage, + 'DBInstanceClass' : instance_class, + 'Engine' : engine, + 'MasterUsername' : master_username, + 'MasterUserPassword' : master_password} + if port: + params['Port'] = port + if db_name: + params['DBName'] = db_name + if param_group: + params['DBParameterGroupName'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if availability_zone: + params['AvailabilityZone'] = availability_zone + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + if multi_az: + params['MultiAZ'] = 'true' + if engine_version: + params['EngineVersion'] = engine_version + if auto_minor_version_upgrade is False: + params['AutoMinorVersionUpgrade'] = 'false' + + return self.get_object('CreateDBInstance', params, DBInstance) + + def create_dbinstance_read_replica(self, id, source_id, + instance_class=None, + port=3306, + availability_zone=None, + auto_minor_version_upgrade=None): + """ + Create a new DBInstance Read Replica. + + :type id: str + :param id: Unique identifier for the new instance. + Must contain 1-63 alphanumeric characters. + First character must be a letter. + May not end with a hyphen or contain two consecutive hyphens + + :type source_id: str + :param source_id: Unique identifier for the DB Instance for which this + DB Instance will act as a Read Replica. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the + DBInstance. Default is to inherit from + the source DB Instance. + + Valid values are: + + * db.m1.small + * db.m1.large + * db.m1.xlarge + * db.m2.xlarge + * db.m2.2xlarge + * db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Default is to inherit from source DB Instance. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :type auto_minor_version_upgrade: bool + :param auto_minor_version_upgrade: Indicates that minor engine + upgrades will be applied + automatically to the Read Replica + during the maintenance window. + Default is to inherit this value + from the source DB Instance. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The new db instance. + """ + params = {'DBInstanceIdentifier' : id, + 'SourceDBInstanceIdentifier' : source_id} + if instance_class: + params['DBInstanceClass'] = instance_class + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + if auto_minor_version_upgrade is not None: + if auto_minor_version_upgrade is True: + params['AutoMinorVersionUpgrade'] = 'true' + else: + params['AutoMinorVersionUpgrade'] = 'false' + + return self.get_object('CreateDBInstanceReadReplica', + params, DBInstance) + + def modify_dbinstance(self, id, param_group=None, security_groups=None, + preferred_maintenance_window=None, + master_password=None, allocated_storage=None, + instance_class=None, + backup_retention_period=None, + preferred_backup_window=None, + multi_az=False, + apply_immediately=False): + """ + Modify an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) + during which maintenance can + occur. + Default is Sun:05:00-Sun:09:00 + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-15 alphanumeric characters. + + :type allocated_storage: int + :param allocated_storage: The new allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the + DBInstance. Changes will be applied at + next maintenance window unless + apply_immediately is True. + + Valid values are: + + * db.m1.small + * db.m1.large + * db.m1.xlarge + * db.m2.xlarge + * db.m2.2xlarge + * db.m2.4xlarge + + :type apply_immediately: bool + :param apply_immediately: If true, the modifications will be applied + as soon as possible rather than waiting for + the next preferred maintenance window. + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :type multi_az: bool + :param multi_az: If True, specifies the DB Instance will be + deployed in multiple availability zones. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The modified db instance. + """ + params = {'DBInstanceIdentifier' : id} + if param_group: + params['DBParameterGroupName'] = param_group + if security_groups: + l = [] + for group in security_groups: + if isinstance(group, DBSecurityGroup): + l.append(group.name) + else: + l.append(group) + self.build_list_params(params, l, 'DBSecurityGroups.member') + if preferred_maintenance_window: + params['PreferredMaintenanceWindow'] = preferred_maintenance_window + if master_password: + params['MasterUserPassword'] = master_password + if allocated_storage: + params['AllocatedStorage'] = allocated_storage + if instance_class: + params['DBInstanceClass'] = instance_class + if backup_retention_period: + params['BackupRetentionPeriod'] = backup_retention_period + if preferred_backup_window: + params['PreferredBackupWindow'] = preferred_backup_window + if multi_az: + params['MultiAZ'] = 'true' + if apply_immediately: + params['ApplyImmediately'] = 'true' + + return self.get_object('ModifyDBInstance', params, DBInstance) + + def delete_dbinstance(self, id, skip_final_snapshot=False, + final_snapshot_id=''): + """ + Delete an existing DBInstance. + + :type id: str + :param id: Unique identifier for the new instance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + params = {'DBInstanceIdentifier' : id} + if skip_final_snapshot: + params['SkipFinalSnapshot'] = 'true' + else: + params['SkipFinalSnapshot'] = 'false' + params['FinalDBSnapshotIdentifier'] = final_snapshot_id + return self.get_object('DeleteDBInstance', params, DBInstance) + + + def reboot_dbinstance(self, id): + """ + Reboot DBInstance. + + :type id: str + :param id: Unique identifier of the instance. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The rebooting db instance. + """ + params = {'DBInstanceIdentifier' : id} + return self.get_object('RebootDBInstance', params, DBInstance) + + # DBParameterGroup methods + + def get_all_dbparameter_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all parameter groups associated with your account in a region. + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + If not provided, all DBParameter groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.ec2.parametergroup.ParameterGroup` + """ + params = {} + if groupname: + params['DBParameterGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBParameterGroups', params, + [('DBParameterGroup', ParameterGroup)]) + + def get_all_dbparameters(self, groupname, source=None, + max_records=None, marker=None): + """ + Get all parameters associated with a ParameterGroup + + :type groupname: str + :param groupname: The name of the DBParameter group to retrieve. + + :type source: str + :param source: Specifies which parameters to return. + If not specified, all parameters will be returned. + Valid values are: user|system|engine-default + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: :class:`boto.ec2.parametergroup.ParameterGroup` + :return: The ParameterGroup + """ + params = {'DBParameterGroupName' : groupname} + if source: + params['Source'] = source + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + pg = self.get_object('DescribeDBParameters', params, ParameterGroup) + pg.name = groupname + return pg + + def create_parameter_group(self, name, engine='MySQL5.1', description=''): + """ + Create a new dbparameter group for your account. + + :type name: string + :param name: The name of the new dbparameter group + + :type engine: str + :param engine: Name of database engine. Must be MySQL5.1 for now. + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBParameterGroupName': name, + 'Engine': engine, + 'Description' : description} + return self.get_object('CreateDBParameterGroup', params, ParameterGroup) + + def modify_parameter_group(self, name, parameters=None): + """ + Modify a parameter group for your account. + + :type name: string + :param name: The name of the new parameter group + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The new parameters + + :rtype: :class:`boto.rds.parametergroup.ParameterGroup` + :return: The newly created ParameterGroup + """ + params = {'DBParameterGroupName': name} + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_list('ModifyDBParameterGroup', params, ParameterGroup) + + def reset_parameter_group(self, name, reset_all_params=False, parameters=None): + """ + Resets some or all of the parameters of a ParameterGroup to the + default value + + :type key_name: string + :param key_name: The name of the ParameterGroup to reset + + :type parameters: list of :class:`boto.rds.parametergroup.Parameter` + :param parameters: The parameters to reset. If not supplied, all parameters + will be reset. + """ + params = {'DBParameterGroupName':name} + if reset_all_params: + params['ResetAllParameters'] = 'true' + else: + params['ResetAllParameters'] = 'false' + for i in range(0, len(parameters)): + parameter = parameters[i] + parameter.merge(params, i+1) + return self.get_status('ResetDBParameterGroup', params) + + def delete_parameter_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBParameterGroupName':name} + return self.get_status('DeleteDBParameterGroup', params) + + # DBSecurityGroup methods + + def get_all_dbsecurity_groups(self, groupname=None, max_records=None, + marker=None): + """ + Get all security groups associated with your account in a region. + + :type groupnames: list + :param groupnames: A list of the names of security groups to retrieve. + If not provided, all security groups will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + """ + params = {} + if groupname: + params['DBSecurityGroupName'] = groupname + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSecurityGroups', params, + [('DBSecurityGroup', DBSecurityGroup)]) + + def create_dbsecurity_group(self, name, description=None): + """ + Create a new security group for your account. + This will create the security group within the region you + are currently connected to. + + :type name: string + :param name: The name of the new security group + + :type description: string + :param description: The description of the new security group + + :rtype: :class:`boto.rds.dbsecuritygroup.DBSecurityGroup` + :return: The newly created DBSecurityGroup + """ + params = {'DBSecurityGroupName':name} + if description: + params['DBSecurityGroupDescription'] = description + group = self.get_object('CreateDBSecurityGroup', params, DBSecurityGroup) + group.name = name + group.description = description + return group + + def delete_dbsecurity_group(self, name): + """ + Delete a DBSecurityGroup from your account. + + :type key_name: string + :param key_name: The name of the DBSecurityGroup to delete + """ + params = {'DBSecurityGroupName':name} + return self.get_status('DeleteDBSecurityGroup', params) + + def authorize_dbsecurity_group(self, group_name, cidr_ip=None, + ec2_security_group_name=None, + ec2_security_group_owner_id=None): + """ + Add a new rule to an existing security group. + You need to pass in either src_security_group_name and + src_security_group_owner_id OR a CIDR block but not both. + + :type group_name: string + :param group_name: The name of the security group you are adding + the rule to. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group you are + granting access to. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + group you are granting access to. + + :type cidr_ip: string + :param cidr_ip: The CIDR block you are providing access to. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = urllib.quote(cidr_ip) + return self.get_object('AuthorizeDBSecurityGroupIngress', params, DBSecurityGroup) + + def revoke_dbsecurity_group(self, group_name, ec2_security_group_name=None, + ec2_security_group_owner_id=None, cidr_ip=None): + """ + Remove an existing rule from an existing security group. + You need to pass in either ec2_security_group_name and + ec2_security_group_owner_id OR a CIDR block. + + :type group_name: string + :param group_name: The name of the security group you are removing + the rule from. + + :type ec2_security_group_name: string + :param ec2_security_group_name: The name of the EC2 security group from which + you are removing access. + + :type ec2_security_group_owner_id: string + :param ec2_security_group_owner_id: The ID of the owner of the EC2 security + from which you are removing access. + + :type cidr_ip: string + :param cidr_ip: The CIDR block from which you are removing access. + See http://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing + + :rtype: bool + :return: True if successful. + """ + params = {'DBSecurityGroupName':group_name} + if ec2_security_group_name: + params['EC2SecurityGroupName'] = ec2_security_group_name + if ec2_security_group_owner_id: + params['EC2SecurityGroupOwnerId'] = ec2_security_group_owner_id + if cidr_ip: + params['CIDRIP'] = cidr_ip + return self.get_object('RevokeDBSecurityGroupIngress', params, DBSecurityGroup) + + # For backwards compatibility. This method was improperly named + # in previous versions. I have renamed it to match the others. + revoke_security_group = revoke_dbsecurity_group + + # DBSnapshot methods + + def get_all_dbsnapshots(self, snapshot_id=None, instance_id=None, + max_records=None, marker=None): + """ + Get information about DB Snapshots. + + :type snapshot_id: str + :param snapshot_id: The unique identifier of an RDS snapshot. + If not provided, all RDS snapshots will be returned. + + :type instance_id: str + :param instance_id: The identifier of a DBInstance. If provided, + only the DBSnapshots related to that instance will + be returned. + If not provided, all RDS snapshots will be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of :class:`boto.rds.dbsnapshot.DBSnapshot` + """ + params = {} + if snapshot_id: + params['DBSnapshotIdentifier'] = snapshot_id + if instance_id: + params['DBInstanceIdentifier'] = instance_id + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeDBSnapshots', params, + [('DBSnapshot', DBSnapshot)]) + + def create_dbsnapshot(self, snapshot_id, dbinstance_id): + """ + Create a new DB snapshot. + + :type snapshot_id: string + :param snapshot_id: The identifier for the DBSnapshot + + :type dbinstance_id: string + :param dbinstance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + params = {'DBSnapshotIdentifier' : snapshot_id, + 'DBInstanceIdentifier' : dbinstance_id} + return self.get_object('CreateDBSnapshot', params, DBSnapshot) + + def delete_dbsnapshot(self, identifier): + """ + Delete a DBSnapshot + + :type identifier: string + :param identifier: The identifier of the DBSnapshot to delete + """ + params = {'DBSnapshotIdentifier' : identifier} + return self.get_object('DeleteDBSnapshot', params, DBSnapshot) + + def restore_dbinstance_from_dbsnapshot(self, identifier, instance_id, + instance_class, port=None, + availability_zone=None): + + """ + Create a new DBInstance from a DB snapshot. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :type instance_id: string + :param instance_id: The source identifier for the RDS instance from + which the snapshot is created. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'DBSnapshotIdentifier' : identifier, + 'DBInstanceIdentifier' : instance_id, + 'DBInstanceClass' : instance_class} + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceFromDBSnapshot', + params, DBInstance) + + def restore_dbinstance_from_point_in_time(self, source_instance_id, + target_instance_id, + use_latest=False, + restore_time=None, + dbinstance_class=None, + port=None, + availability_zone=None): + + """ + Create a new DBInstance from a point in time. + + :type source_instance_id: string + :param source_instance_id: The identifier for the source DBInstance. + + :type target_instance_id: string + :param target_instance_id: The identifier of the new DBInstance. + + :type use_latest: bool + :param use_latest: If True, the latest snapshot availabile will + be used. + + :type restore_time: datetime + :param restore_time: The date and time to restore from. Only + used if use_latest is False. + + :type instance_class: str + :param instance_class: The compute and memory capacity of the DBInstance. + Valid values are: + db.m1.small | db.m1.large | db.m1.xlarge | + db.m2.2xlarge | db.m2.4xlarge + + :type port: int + :param port: Port number on which database accepts connections. + Valid values [1115-65535]. Defaults to 3306. + + :type availability_zone: str + :param availability_zone: Name of the availability zone to place + DBInstance into. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The newly created DBInstance + """ + params = {'SourceDBInstanceIdentifier' : source_instance_id, + 'TargetDBInstanceIdentifier' : target_instance_id} + if use_latest: + params['UseLatestRestorableTime'] = 'true' + elif restore_time: + params['RestoreTime'] = restore_time.isoformat() + if dbinstance_class: + params['DBInstanceClass'] = dbinstance_class + if port: + params['Port'] = port + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('RestoreDBInstanceToPointInTime', + params, DBInstance) + + # Events + + def get_all_events(self, source_identifier=None, source_type=None, + start_time=None, end_time=None, + max_records=None, marker=None): + """ + Get information about events related to your DBInstances, + DBSecurityGroups and DBParameterGroups. + + :type source_identifier: str + :param source_identifier: If supplied, the events returned will be + limited to those that apply to the identified + source. The value of this parameter depends + on the value of source_type. If neither + parameter is specified, all events in the time + span will be returned. + + :type source_type: str + :param source_type: Specifies how the source_identifier should + be interpreted. Valid values are: + b-instance | db-security-group | + db-parameter-group | db-snapshot + + :type start_time: datetime + :param start_time: The beginning of the time interval for events. + If not supplied, all available events will + be returned. + + :type end_time: datetime + :param end_time: The ending of the time interval for events. + If not supplied, all available events will + be returned. + + :type max_records: int + :param max_records: The maximum number of records to be returned. + If more results are available, a MoreToken will + be returned in the response that can be used to + retrieve additional records. Default is 100. + + :type marker: str + :param marker: The marker provided by a previous request. + + :rtype: list + :return: A list of class:`boto.rds.event.Event` + """ + params = {} + if source_identifier and source_type: + params['SourceIdentifier'] = source_identifier + params['SourceType'] = source_type + if start_time: + params['StartTime'] = start_time.isoformat() + if end_time: + params['EndTime'] = end_time.isoformat() + if max_records: + params['MaxRecords'] = max_records + if marker: + params['Marker'] = marker + return self.get_list('DescribeEvents', params, [('Event', Event)]) + + diff --git a/backup/src/boto/rds/dbinstance.py b/backup/src/boto/rds/dbinstance.py new file mode 100644 index 0000000..02f9af6 --- /dev/null +++ b/backup/src/boto/rds/dbinstance.py @@ -0,0 +1,264 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.rds.dbsecuritygroup import DBSecurityGroup +from boto.rds.parametergroup import ParameterGroup + +class DBInstance(object): + """ + Represents a RDS DBInstance + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.create_time = None + self.engine = None + self.status = None + self.allocated_storage = None + self.endpoint = None + self.instance_class = None + self.master_username = None + self.parameter_group = None + self.security_group = None + self.availability_zone = None + self.backup_retention_period = None + self.preferred_backup_window = None + self.preferred_maintenance_window = None + self.latest_restorable_time = None + self.multi_az = False + self.pending_modified_values = None + self._in_endpoint = False + self._port = None + self._address = None + + def __repr__(self): + return 'DBInstance:%s' % self.id + + def startElement(self, name, attrs, connection): + if name == 'Endpoint': + self._in_endpoint = True + elif name == 'DBParameterGroup': + self.parameter_group = ParameterGroup(self.connection) + return self.parameter_group + elif name == 'DBSecurityGroup': + self.security_group = DBSecurityGroup(self.connection) + return self.security_group + elif name == 'PendingModifiedValues': + self.pending_modified_values = PendingModifiedValues() + return self.pending_modified_values + return None + + def endElement(self, name, value, connection): + if name == 'DBInstanceIdentifier': + self.id = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'InstanceCreateTime': + self.create_time = value + elif name == 'Engine': + self.engine = value + elif name == 'DBInstanceStatus': + self.status = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'DBInstanceClass': + self.instance_class = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'Port': + if self._in_endpoint: + self._port = int(value) + elif name == 'Address': + if self._in_endpoint: + self._address = value + elif name == 'Endpoint': + self.endpoint = (self._address, self._port) + self._in_endpoint = False + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'BackupRetentionPeriod': + self.backup_retention_period = value + elif name == 'LatestRestorableTime': + self.latest_restorable_time = value + elif name == 'PreferredMaintenanceWindow': + self.preferred_maintenance_window = value + elif name == 'PreferredBackupWindow': + self.preferred_backup_window = value + elif name == 'MultiAZ': + if value.lower() == 'true': + self.multi_az = True + else: + setattr(self, name, value) + + def snapshot(self, snapshot_id): + """ + Create a new DB snapshot of this DBInstance. + + :type identifier: string + :param identifier: The identifier for the DBSnapshot + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + return self.connection.create_dbsnapshot(snapshot_id, self.id) + + def reboot(self): + """ + Reboot this DBInstance + + :rtype: :class:`boto.rds.dbsnapshot.DBSnapshot` + :return: The newly created DBSnapshot + """ + return self.connection.reboot_dbinstance(self.id) + + def update(self, validate=False): + """ + Update the DB instance's status information by making a call to fetch + the current instance attributes from the service. + + :type validate: bool + :param validate: By default, if EC2 returns no data about the + instance the update method returns quietly. If + the validate param is True, however, it will + raise a ValueError exception if no data is + returned from EC2. + """ + rs = self.connection.get_all_dbinstances(self.id) + if len(rs) > 0: + for i in rs: + if i.id == self.id: + self.__dict__.update(i.__dict__) + elif validate: + raise ValueError('%s is not a valid Instance ID' % self.id) + return self.status + + + def stop(self, skip_final_snapshot=False, final_snapshot_id=''): + """ + Delete this DBInstance. + + :type skip_final_snapshot: bool + :param skip_final_snapshot: This parameter determines whether a final + db snapshot is created before the instance + is deleted. If True, no snapshot is created. + If False, a snapshot is created before + deleting the instance. + + :type final_snapshot_id: str + :param final_snapshot_id: If a final snapshot is requested, this + is the identifier used for that snapshot. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The deleted db instance. + """ + return self.connection.delete_dbinstance(self.id, + skip_final_snapshot, + final_snapshot_id) + + def modify(self, param_group=None, security_groups=None, + preferred_maintenance_window=None, + master_password=None, allocated_storage=None, + instance_class=None, + backup_retention_period=None, + preferred_backup_window=None, + multi_az=False, + apply_immediately=False): + """ + Modify this DBInstance. + + :type security_groups: list of str or list of DBSecurityGroup objects + :param security_groups: List of names of DBSecurityGroup to authorize on + this DBInstance. + + :type preferred_maintenance_window: str + :param preferred_maintenance_window: The weekly time range (in UTC) + during which maintenance can + occur. + Default is Sun:05:00-Sun:09:00 + + :type master_password: str + :param master_password: Password of master user for the DBInstance. + Must be 4-15 alphanumeric characters. + + :type allocated_storage: int + :param allocated_storage: The new allocated storage size, in GBs. + Valid values are [5-1024] + + :type instance_class: str + :param instance_class: The compute and memory capacity of the + DBInstance. Changes will be applied at + next maintenance window unless + apply_immediately is True. + + Valid values are: + + * db.m1.small + * db.m1.large + * db.m1.xlarge + * db.m2.xlarge + * db.m2.2xlarge + * db.m2.4xlarge + + :type apply_immediately: bool + :param apply_immediately: If true, the modifications will be applied + as soon as possible rather than waiting for + the next preferred maintenance window. + + :type backup_retention_period: int + :param backup_retention_period: The number of days for which automated + backups are retained. Setting this to + zero disables automated backups. + + :type preferred_backup_window: str + :param preferred_backup_window: The daily time range during which + automated backups are created (if + enabled). Must be in h24:mi-hh24:mi + format (UTC). + + :type multi_az: bool + :param multi_az: If True, specifies the DB Instance will be + deployed in multiple availability zones. + + :rtype: :class:`boto.rds.dbinstance.DBInstance` + :return: The modified db instance. + """ + return self.connection.modify_dbinstance(self.id, + param_group, + security_groups, + preferred_maintenance_window, + master_password, + allocated_storage, + instance_class, + backup_retention_period, + preferred_backup_window, + multi_az, + apply_immediately) + +class PendingModifiedValues(dict): + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name != 'PendingModifiedValues': + self[name] = value + diff --git a/backup/src/boto/rds/dbsecuritygroup.py b/backup/src/boto/rds/dbsecuritygroup.py new file mode 100644 index 0000000..1555ca0 --- /dev/null +++ b/backup/src/boto/rds/dbsecuritygroup.py @@ -0,0 +1,160 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an DBSecurityGroup +""" +from boto.ec2.securitygroup import SecurityGroup + +class DBSecurityGroup(object): + + def __init__(self, connection=None, owner_id=None, + name=None, description=None): + self.connection = connection + self.owner_id = owner_id + self.name = name + self.description = description + self.ec2_groups = [] + self.ip_ranges = [] + + def __repr__(self): + return 'DBSecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'IPRange': + cidr = IPRange(self) + self.ip_ranges.append(cidr) + return cidr + elif name == 'EC2SecurityGroup': + ec2_grp = EC2SecurityGroup(self) + self.ec2_groups.append(ec2_grp) + return ec2_grp + else: + return None + + def endElement(self, name, value, connection): + if name == 'OwnerId': + self.owner_id = value + elif name == 'DBSecurityGroupName': + self.name = value + elif name == 'DBSecurityGroupDescription': + self.description = value + elif name == 'IPRanges': + pass + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_dbsecurity_group(self.name) + + def authorize(self, cidr_ip=None, ec2_group=None): + """ + Add a new rule to this DBSecurity group. + You need to pass in either a CIDR block to authorize or + and EC2 SecurityGroup. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to authorize + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + else: + group_name = None + group_owner_id = None + return self.connection.authorize_dbsecurity_group(self.name, + cidr_ip, + group_name, + group_owner_id) + + def revoke(self, cidr_ip=None, ec2_group=None): + """ + Revoke access to a CIDR range or EC2 SecurityGroup. + You need to pass in either a CIDR block or + an EC2 SecurityGroup from which to revoke access. + + @type cidr_ip: string + @param cidr_ip: A valid CIDR IP range to revoke + + @type ec2_group: :class:`boto.ec2.securitygroup.SecurityGroup>` + + @rtype: bool + @return: True if successful. + """ + if isinstance(ec2_group, SecurityGroup): + group_name = ec2_group.name + group_owner_id = ec2_group.owner_id + return self.connection.revoke_dbsecurity_group( + self.name, + ec2_security_group_name=group_name, + ec2_security_group_owner_id=group_owner_id) + + # Revoking by CIDR IP range + return self.connection.revoke_dbsecurity_group( + self.name, cidr_ip=cidr_ip) + +class IPRange(object): + + def __init__(self, parent=None): + self.parent = parent + self.cidr_ip = None + self.status = None + + def __repr__(self): + return 'IPRange:%s' % self.cidr_ip + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'CIDRIP': + self.cidr_ip = value + elif name == 'Status': + self.status = value + else: + setattr(self, name, value) + +class EC2SecurityGroup(object): + + def __init__(self, parent=None): + self.parent = parent + self.name = None + self.owner_id = None + + def __repr__(self): + return 'EC2SecurityGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'EC2SecurityGroupName': + self.name = value + elif name == 'EC2SecurityGroupOwnerId': + self.owner_id = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/rds/dbsnapshot.py b/backup/src/boto/rds/dbsnapshot.py new file mode 100644 index 0000000..78d0230 --- /dev/null +++ b/backup/src/boto/rds/dbsnapshot.py @@ -0,0 +1,74 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class DBSnapshot(object): + """ + Represents a RDS DB Snapshot + """ + + def __init__(self, connection=None, id=None): + self.connection = connection + self.id = id + self.engine = None + self.snapshot_create_time = None + self.instance_create_time = None + self.port = None + self.status = None + self.availability_zone = None + self.master_username = None + self.allocated_storage = None + self.instance_id = None + self.availability_zone = None + + def __repr__(self): + return 'DBSnapshot:%s' % self.id + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Engine': + self.engine = value + elif name == 'InstanceCreateTime': + self.instance_create_time = value + elif name == 'SnapshotCreateTime': + self.snapshot_create_time = value + elif name == 'DBInstanceIdentifier': + self.instance_id = value + elif name == 'DBSnapshotIdentifier': + self.id = value + elif name == 'Port': + self.port = int(value) + elif name == 'Status': + self.status = value + elif name == 'AvailabilityZone': + self.availability_zone = value + elif name == 'MasterUsername': + self.master_username = value + elif name == 'AllocatedStorage': + self.allocated_storage = int(value) + elif name == 'SnapshotTime': + self.time = value + else: + setattr(self, name, value) + + + diff --git a/backup/src/boto/rds/event.py b/backup/src/boto/rds/event.py new file mode 100644 index 0000000..a91f8f0 --- /dev/null +++ b/backup/src/boto/rds/event.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Event(object): + + def __init__(self, connection=None): + self.connection = connection + self.message = None + self.source_identifier = None + self.source_type = None + self.engine = None + self.date = None + + def __repr__(self): + return '"%s"' % self.message + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'SourceIdentifier': + self.source_identifier = value + elif name == 'SourceType': + self.source_type = value + elif name == 'Message': + self.message = value + elif name == 'Date': + self.date = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/rds/parametergroup.py b/backup/src/boto/rds/parametergroup.py new file mode 100644 index 0000000..44d00e2 --- /dev/null +++ b/backup/src/boto/rds/parametergroup.py @@ -0,0 +1,201 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ParameterGroup(dict): + + def __init__(self, connection=None): + dict.__init__(self) + self.connection = connection + self.name = None + self.description = None + self.engine = None + self._current_param = None + + def __repr__(self): + return 'ParameterGroup:%s' % self.name + + def startElement(self, name, attrs, connection): + if name == 'Parameter': + if self._current_param: + self[self._current_param.name] = self._current_param + self._current_param = Parameter(self) + return self._current_param + + def endElement(self, name, value, connection): + if name == 'DBParameterGroupName': + self.name = value + elif name == 'Description': + self.description = value + elif name == 'Engine': + self.engine = value + else: + setattr(self, name, value) + + def modifiable(self): + mod = [] + for key in self: + p = self[key] + if p.is_modifiable: + mod.append(p) + return mod + + def get_params(self): + pg = self.connection.get_all_dbparameters(self.name) + self.update(pg) + + def add_param(self, name, value, apply_method): + param = Parameter() + param.name = name + param.value = value + param.apply_method = apply_method + self.params.append(param) + +class Parameter(object): + """ + Represents a RDS Parameter + """ + + ValidTypes = {'integer' : int, + 'string' : str, + 'boolean' : bool} + ValidSources = ['user', 'system', 'engine-default'] + ValidApplyTypes = ['static', 'dynamic'] + ValidApplyMethods = ['immediate', 'pending-reboot'] + + def __init__(self, group=None, name=None): + self.group = group + self.name = name + self._value = None + self.type = str + self.source = None + self.is_modifiable = True + self.description = None + self.apply_method = None + self.allowed_values = None + + def __repr__(self): + return 'Parameter:%s' % self.name + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'ParameterName': + self.name = value + elif name == 'ParameterValue': + self._value = value + elif name == 'DataType': + if value in self.ValidTypes: + self.type = value + elif name == 'Source': + if value in self.ValidSources: + self.source = value + elif name == 'IsModifiable': + if value.lower() == 'true': + self.is_modifiable = True + else: + self.is_modifiable = False + elif name == 'Description': + self.description = value + elif name == 'ApplyType': + if value in self.ValidApplyTypes: + self.apply_type = value + elif name == 'AllowedValues': + self.allowed_values = value + else: + setattr(self, name, value) + + def merge(self, d, i): + prefix = 'Parameters.member.%d.' % i + if self.name: + d[prefix+'ParameterName'] = self.name + if self._value: + d[prefix+'ParameterValue'] = self._value + if self.apply_type: + d[prefix+'ApplyMethod'] = self.apply_method + + def _set_string_value(self, value): + if not isinstance(value, str) or isinstance(value, unicode): + raise ValueError, 'value must be of type str' + if self.allowed_values: + choices = self.allowed_values.split(',') + if value not in choices: + raise ValueError, 'value must be in %s' % self.allowed_values + self._value = value + + def _set_integer_value(self, value): + if isinstance(value, str) or isinstance(value, unicode): + value = int(value) + if isinstance(value, int) or isinstance(value, long): + if self.allowed_values: + min, max = self.allowed_values.split('-') + if value < int(min) or value > int(max): + raise ValueError, 'range is %s' % self.allowed_values + self._value = value + else: + raise ValueError, 'value must be integer' + + def _set_boolean_value(self, value): + if isinstance(value, bool): + self._value = value + elif isinstance(value, str) or isinstance(value, unicode): + if value.lower() == 'true': + self._value = True + else: + self._value = False + else: + raise ValueError, 'value must be boolean' + + def set_value(self, value): + if self.type == 'string': + self._set_string_value(value) + elif self.type == 'integer': + self._set_integer_value(value) + elif self.type == 'boolean': + self._set_boolean_value(value) + else: + raise TypeError, 'unknown type (%s)' % self.type + + def get_value(self): + if self._value == None: + return self._value + if self.type == 'string': + return self._value + elif self.type == 'integer': + if not isinstance(self._value, int) and not isinstance(self._value, long): + self._set_integer_value(self._value) + return self._value + elif self.type == 'boolean': + if not isinstance(self._value, bool): + self._set_boolean_value(self._value) + return self._value + else: + raise TypeError, 'unknown type (%s)' % self.type + + value = property(get_value, set_value, 'The value of the parameter') + + def apply(self, immediate=False): + if immediate: + self.apply_method = 'immediate' + else: + self.apply_method = 'pending-reboot' + self.group.connection.modify_parameter_group(self.group.name, [self]) + diff --git a/backup/src/boto/rds/regioninfo.py b/backup/src/boto/rds/regioninfo.py new file mode 100644 index 0000000..7d186ae --- /dev/null +++ b/backup/src/boto/rds/regioninfo.py @@ -0,0 +1,32 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.regioninfo import RegionInfo + +class RDSRegionInfo(RegionInfo): + + def __init__(self, connection=None, name=None, endpoint=None): + from boto.rds import RDSConnection + RegionInfo.__init__(self, connection, name, endpoint, + RDSConnection) diff --git a/backup/src/boto/regioninfo.py b/backup/src/boto/regioninfo.py new file mode 100644 index 0000000..907385f --- /dev/null +++ b/backup/src/boto/regioninfo.py @@ -0,0 +1,64 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class RegionInfo(object): + """ + Represents an AWS Region + """ + + def __init__(self, connection=None, name=None, endpoint=None, + connection_cls=None): + self.connection = connection + self.name = name + self.endpoint = endpoint + self.connection_cls = connection_cls + + def __repr__(self): + return 'RegionInfo:%s' % self.name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'regionName': + self.name = value + elif name == 'regionEndpoint': + self.endpoint = value + else: + setattr(self, name, value) + + def connect(self, **kw_params): + """ + Connect to this Region's endpoint. Returns an connection + object pointing to the endpoint associated with this region. + You may pass any of the arguments accepted by the connection + class's constructor as keyword arguments and they will be + passed along to the connection object. + + :rtype: Connection object + :return: The connection to this regions endpoint + """ + if self.connection_cls: + return self.connection_cls(region=self, **kw_params) + + diff --git a/backup/src/boto/resultset.py b/backup/src/boto/resultset.py new file mode 100644 index 0000000..075fc5e --- /dev/null +++ b/backup/src/boto/resultset.py @@ -0,0 +1,153 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class ResultSet(list): + """ + The ResultSet is used to pass results back from the Amazon services + to the client. It is light wrapper around Python's :py:class:`list` class, + with some additional methods for parsing XML results from AWS. + Because I don't really want any dependencies on external libraries, + I'm using the standard SAX parser that comes with Python. The good news is + that it's quite fast and efficient but it makes some things rather + difficult. + + You can pass in, as the marker_elem parameter, a list of tuples. + Each tuple contains a string as the first element which represents + the XML element that the resultset needs to be on the lookout for + and a Python class as the second element of the tuple. Each time the + specified element is found in the XML, a new instance of the class + will be created and popped onto the stack. + + :ivar str next_token: A hash used to assist in paging through very long + result sets. In most cases, passing this value to certain methods + will give you another 'page' of results. + """ + def __init__(self, marker_elem=None): + list.__init__(self) + if isinstance(marker_elem, list): + self.markers = marker_elem + else: + self.markers = [] + self.marker = None + self.key_marker = None + self.next_key_marker = None + self.next_version_id_marker = None + self.version_id_marker = None + self.is_truncated = False + self.next_token = None + self.status = True + + def startElement(self, name, attrs, connection): + for t in self.markers: + if name == t[0]: + obj = t[1](connection) + self.append(obj) + return obj + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'IsTruncated': + self.is_truncated = self.to_boolean(value) + elif name == 'Marker': + self.marker = value + elif name == 'KeyMarker': + self.key_marker = value + elif name == 'NextKeyMarker': + self.next_key_marker = value + elif name == 'VersionIdMarker': + self.version_id_marker = value + elif name == 'NextVersionIdMarker': + self.next_version_id_marker = value + elif name == 'UploadIdMarker': + self.upload_id_marker = value + elif name == 'NextUploadIdMarker': + self.next_upload_id_marker = value + elif name == 'Bucket': + self.bucket = value + elif name == 'MaxUploads': + self.max_uploads = int(value) + elif name == 'Prefix': + self.prefix = value + elif name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'ItemName': + self.append(value) + elif name == 'NextToken': + self.next_token = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + else: + setattr(self, name, value) + +class BooleanResult(object): + + def __init__(self, marker_elem=None): + self.status = True + self.request_id = None + self.box_usage = None + + def __repr__(self): + if self.status: + return 'True' + else: + return 'False' + + def __nonzero__(self): + return self.status + + def startElement(self, name, attrs, connection): + return None + + def to_boolean(self, value, true_value='true'): + if value == true_value: + return True + else: + return False + + def endElement(self, name, value, connection): + if name == 'return': + self.status = self.to_boolean(value) + elif name == 'StatusCode': + self.status = self.to_boolean(value, 'Success') + elif name == 'IsValid': + self.status = self.to_boolean(value, 'True') + elif name == 'RequestId': + self.request_id = value + elif name == 'requestId': + self.request_id = value + elif name == 'BoxUsage': + self.request_id = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/route53/__init__.py b/backup/src/boto/route53/__init__.py new file mode 100644 index 0000000..d404bc7 --- /dev/null +++ b/backup/src/boto/route53/__init__.py @@ -0,0 +1,26 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# this is here for backward compatibility +# originally, the Route53Connection class was defined here +from connection import Route53Connection diff --git a/backup/src/boto/route53/connection.py b/backup/src/boto/route53/connection.py new file mode 100644 index 0000000..bbd218c --- /dev/null +++ b/backup/src/boto/route53/connection.py @@ -0,0 +1,285 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +import xml.sax +import time +import uuid +import urllib +import boto +from boto.connection import AWSAuthConnection +from boto import handler +from boto.resultset import ResultSet +import boto.jsonresponse +import exception +import hostedzone + +HZXML = """ + + %(name)s + %(caller_ref)s + + %(comment)s + +""" + +#boto.set_stream_logger('dns') + +class Route53Connection(AWSAuthConnection): + + DefaultHost = 'route53.amazonaws.com' + Version = '2010-10-01' + XMLNameSpace = 'https://route53.amazonaws.com/doc/2010-10-01/' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0): + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + True, port, proxy, proxy_port, debug=debug) + + def _required_auth_capability(self): + return ['route53'] + + def make_request(self, action, path, headers=None, data='', params=None): + if params: + pairs = [] + for key, val in params.iteritems(): + if val is None: continue + pairs.append(key + '=' + urllib.quote(str(val))) + path += '?' + '&'.join(pairs) + return AWSAuthConnection.make_request(self, action, path, headers, data) + + # Hosted Zones + + def get_all_hosted_zones(self): + """ + Returns a Python data structure with information about all + Hosted Zones defined for the AWS account. + """ + response = self.make_request('GET', '/%s/hostedzone' % self.Version) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise exception.DNSServerError(response.status, + response.reason, + body) + e = boto.jsonresponse.Element(list_marker='HostedZones', + item_marker=('HostedZone',)) + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + + def get_hosted_zone(self, hosted_zone_id): + """ + Get detailed information about a particular Hosted Zone. + + :type hosted_zone_id: str + :param hosted_zone_id: The unique identifier for the Hosted Zone + + """ + uri = '/%s/hostedzone/%s' % (self.Version, hosted_zone_id) + response = self.make_request('GET', uri) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise exception.DNSServerError(response.status, + response.reason, + body) + e = boto.jsonresponse.Element(list_marker='NameServers', + item_marker=('NameServer',)) + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + + def create_hosted_zone(self, domain_name, caller_ref=None, comment=''): + """ + Create a new Hosted Zone. Returns a Python data structure with + information about the newly created Hosted Zone. + + :type domain_name: str + :param domain_name: The name of the domain. This should be a + fully-specified domain, and should end with + a final period as the last label indication. + If you omit the final period, Amazon Route 53 + assumes the domain is relative to the root. + This is the name you have registered with your + DNS registrar. It is also the name you will + delegate from your registrar to the Amazon + Route 53 delegation servers returned in + response to this request.A list of strings + with the image IDs wanted + + :type caller_ref: str + :param caller_ref: A unique string that identifies the request + and that allows failed CreateHostedZone requests + to be retried without the risk of executing the + operation twice. + If you don't provide a value for this, boto will + generate a Type 4 UUID and use that. + + :type comment: str + :param comment: Any comments you want to include about the hosted + zone. + + """ + if caller_ref is None: + caller_ref = str(uuid.uuid4()) + params = {'name' : domain_name, + 'caller_ref' : caller_ref, + 'comment' : comment, + 'xmlns' : self.XMLNameSpace} + xml = HZXML % params + uri = '/%s/hostedzone' % self.Version + response = self.make_request('POST', uri, + {'Content-Type' : 'text/xml'}, xml) + body = response.read() + boto.log.debug(body) + if response.status == 201: + e = boto.jsonresponse.Element(list_marker='NameServers', + item_marker=('NameServer',)) + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + else: + raise exception.DNSServerError(response.status, + response.reason, + body) + + def delete_hosted_zone(self, hosted_zone_id): + uri = '/%s/hostedzone/%s' % (self.Version, hosted_zone_id) + response = self.make_request('DELETE', uri) + body = response.read() + boto.log.debug(body) + if response.status not in (200, 204): + raise exception.DNSServerError(response.status, + response.reason, + body) + e = boto.jsonresponse.Element() + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + + # Resource Record Sets + + def get_all_rrsets(self, hosted_zone_id, type=None, + name=None, maxitems=None): + """ + Retrieve the Resource Record Sets defined for this Hosted Zone. + Returns the raw XML data returned by the Route53 call. + + :type hosted_zone_id: str + :param hosted_zone_id: The unique identifier for the Hosted Zone + + :type type: str + :param type: The type of resource record set to begin the record + listing from. Valid choices are: + + * A + * AAAA + * CNAME + * MX + * NS + * PTR + * SOA + * SPF + * SRV + * TXT + + :type name: str + :param name: The first name in the lexicographic ordering of domain + names to be retrieved + + :type maxitems: int + :param maxitems: The maximum number of records + + """ + from boto.route53.record import ResourceRecordSets + params = {'type': type, 'name': name, 'maxitems': maxitems} + uri = '/%s/hostedzone/%s/rrset' % (self.Version, hosted_zone_id) + response = self.make_request('GET', uri, params=params) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise exception.DNSServerError(response.status, + response.reason, + body) + rs = ResourceRecordSets(connection=self, hosted_zone_id=hosted_zone_id) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def change_rrsets(self, hosted_zone_id, xml_body): + """ + Create or change the authoritative DNS information for this + Hosted Zone. + Returns a Python data structure with information about the set of + changes, including the Change ID. + + :type hosted_zone_id: str + :param hosted_zone_id: The unique identifier for the Hosted Zone + + :type xml_body: str + :param xml_body: The list of changes to be made, defined in the + XML schema defined by the Route53 service. + + """ + uri = '/%s/hostedzone/%s/rrset' % (self.Version, hosted_zone_id) + response = self.make_request('POST', uri, + {'Content-Type' : 'text/xml'}, + xml_body) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise exception.DNSServerError(response.status, + response.reason, + body) + e = boto.jsonresponse.Element() + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + + def get_change(self, change_id): + """ + Get information about a proposed set of changes, as submitted + by the change_rrsets method. + Returns a Python data structure with status information about the + changes. + + :type change_id: str + :param change_id: The unique identifier for the set of changes. + This ID is returned in the response to the + change_rrsets method. + + """ + uri = '/%s/change/%s' % (self.Version, change_id) + response = self.make_request('GET', uri) + body = response.read() + boto.log.debug(body) + if response.status >= 300: + raise exception.DNSServerError(response.status, + response.reason, + body) + e = boto.jsonresponse.Element() + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e diff --git a/backup/src/boto/route53/exception.py b/backup/src/boto/route53/exception.py new file mode 100644 index 0000000..ba41285 --- /dev/null +++ b/backup/src/boto/route53/exception.py @@ -0,0 +1,27 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import BotoServerError + +class DNSServerError(BotoServerError): + + pass diff --git a/backup/src/boto/route53/hostedzone.py b/backup/src/boto/route53/hostedzone.py new file mode 100644 index 0000000..66b79b8 --- /dev/null +++ b/backup/src/boto/route53/hostedzone.py @@ -0,0 +1,56 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +class HostedZone(object): + + def __init__(self, id=None, name=None, owner=None, version=None, + caller_reference=None, config=None): + self.id = id + self.name = name + self.owner = owner + self.version = version + self.caller_reference = caller_reference + self.config = config + + def startElement(self, name, attrs, connection): + if name == 'Config': + self.config = Config() + return self.config + else: + return None + + def endElement(self, name, value, connection): + if name == 'Id': + self.id = value + elif name == 'Name': + self.name = value + elif name == 'Owner': + self.owner = value + elif name == 'Version': + self.version = value + elif name == 'CallerReference': + self.caller_reference = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/route53/record.py b/backup/src/boto/route53/record.py new file mode 100644 index 0000000..24f0482 --- /dev/null +++ b/backup/src/boto/route53/record.py @@ -0,0 +1,152 @@ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +RECORD_TYPES = ['A', 'AAAA', 'TXT', 'CNAME', 'MX', 'PTR', 'SRV', 'SPF'] + +from boto.resultset import ResultSet +class ResourceRecordSets(ResultSet): + + ChangeResourceRecordSetsBody = """ + + + %(comment)s + %(changes)s + + """ + + ChangeXML = """ + %(action)s + %(record)s + """ + + + def __init__(self, connection=None, hosted_zone_id=None, comment=None): + self.connection = connection + self.hosted_zone_id = hosted_zone_id + self.comment = comment + self.changes = [] + self.next_record_name = None + self.next_record_type = None + ResultSet.__init__(self, [('ResourceRecordSet', Record)]) + + def __repr__(self): + return '' % self.hosted_zone_id + + def add_change(self, action, name, type, ttl=600): + """Add a change request""" + change = Record(name, type, ttl) + self.changes.append([action, change]) + return change + + def to_xml(self): + """Convert this ResourceRecordSet into XML + to be saved via the ChangeResourceRecordSetsRequest""" + changesXML = "" + for change in self.changes: + changeParams = {"action": change[0], "record": change[1].to_xml()} + changesXML += self.ChangeXML % changeParams + params = {"comment": self.comment, "changes": changesXML} + return self.ChangeResourceRecordSetsBody % params + + def commit(self): + """Commit this change""" + if not self.connection: + import boto + self.connection = boto.connect_route53() + return self.connection.change_rrsets(self.hosted_zone_id, self.to_xml()) + + def endElement(self, name, value, connection): + """Overwritten to also add the NextRecordName and + NextRecordType to the base object""" + if name == 'NextRecordName': + self.next_record_name = value + elif name == 'NextRecordType': + self.next_record_type = value + else: + return ResultSet.endElement(self, name, value, connection) + + def __iter__(self): + """Override the next function to support paging""" + results = ResultSet.__iter__(self) + while results: + for obj in results: + yield obj + if self.is_truncated: + self.is_truncated = False + results = self.connection.get_all_rrsets(self.hosted_zone_id, name=self.next_record_name, type=self.next_record_type) + else: + results = None + + + +class Record(object): + """An individual ResourceRecordSet""" + + XMLBody = """ + %(name)s + %(type)s + %(ttl)s + %(records)s + """ + + ResourceRecordBody = """ + %s + """ + + + def __init__(self, name=None, type=None, ttl=600, resource_records=None): + self.name = name + self.type = type + self.ttl = ttl + if resource_records == None: + resource_records = [] + self.resource_records = resource_records + + def add_value(self, value): + """Add a resource record value""" + self.resource_records.append(value) + + def to_xml(self): + """Spit this resource record set out as XML""" + records = "" + for r in self.resource_records: + records += self.ResourceRecordBody % r + params = { + "name": self.name, + "type": self.type, + "ttl": self.ttl, + "records": records + } + return self.XMLBody % params + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'Type': + self.type = value + elif name == 'TTL': + self.ttl = value + elif name == 'Value': + self.resource_records.append(value) + + def startElement(self, name, attrs, connection): + return None diff --git a/backup/src/boto/s3/__init__.py b/backup/src/boto/s3/__init__.py new file mode 100644 index 0000000..f3f4c1e --- /dev/null +++ b/backup/src/boto/s3/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + diff --git a/backup/src/boto/s3/acl.py b/backup/src/boto/s3/acl.py new file mode 100644 index 0000000..2640499 --- /dev/null +++ b/backup/src/boto/s3/acl.py @@ -0,0 +1,163 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.user import User + + +CannedACLStrings = ['private', 'public-read', + 'public-read-write', 'authenticated-read', + 'bucket-owner-read', 'bucket-owner-full-control'] + + +class Policy: + + def __init__(self, parent=None): + self.parent = parent + self.acl = None + + def __repr__(self): + grants = [] + for g in self.acl.grants: + if g.id == self.owner.id: + grants.append("%s (owner) = %s" % (g.display_name, g.permission)) + else: + if g.type == 'CanonicalUser': + u = g.display_name + elif g.type == 'Group': + u = g.uri + else: + u = g.email + grants.append("%s = %s" % (u, g.permission)) + return "" % ", ".join(grants) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + elif name == 'AccessControlList': + self.acl = ACL(self) + return self.acl + else: + return None + + def endElement(self, name, value, connection): + if name == 'Owner': + pass + elif name == 'AccessControlList': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += self.owner.to_xml() + s += self.acl.to_xml() + s += '' + return s + +class ACL: + + def __init__(self, policy=None): + self.policy = policy + self.grants = [] + + def add_grant(self, grant): + self.grants.append(grant) + + def add_email_grant(self, permission, email_address): + grant = Grant(permission=permission, type='AmazonCustomerByEmail', + email_address=email_address) + self.grants.append(grant) + + def add_user_grant(self, permission, user_id): + grant = Grant(permission=permission, type='CanonicalUser', id=user_id) + self.grants.append(grant) + + def startElement(self, name, attrs, connection): + if name == 'Grant': + self.grants.append(Grant(self)) + return self.grants[-1] + else: + return None + + def endElement(self, name, value, connection): + if name == 'Grant': + pass + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + for grant in self.grants: + s += grant.to_xml() + s += '' + return s + +class Grant: + + NameSpace = 'xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"' + + def __init__(self, permission=None, type=None, id=None, + display_name=None, uri=None, email_address=None): + self.permission = permission + self.id = id + self.display_name = display_name + self.uri = uri + self.email_address = email_address + self.type = type + + def startElement(self, name, attrs, connection): + if name == 'Grantee': + self.type = attrs['xsi:type'] + return None + + def endElement(self, name, value, connection): + if name == 'ID': + self.id = value + elif name == 'DisplayName': + self.display_name = value + elif name == 'URI': + self.uri = value + elif name == 'EmailAddress': + self.email_address = value + elif name == 'Grantee': + pass + elif name == 'Permission': + self.permission = value + else: + setattr(self, name, value) + + def to_xml(self): + s = '' + s += '' % (self.NameSpace, self.type) + if self.type == 'CanonicalUser': + s += '%s' % self.id + s += '%s' % self.display_name + elif self.type == 'Group': + s += '%s' % self.uri + else: + s += '%s' % self.email_address + s += '' + s += '%s' % self.permission + s += '' + return s + + diff --git a/backup/src/boto/s3/bucket.py b/backup/src/boto/s3/bucket.py new file mode 100644 index 0000000..6188196 --- /dev/null +++ b/backup/src/boto/s3/bucket.py @@ -0,0 +1,1068 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto import handler +from boto.provider import Provider +from boto.resultset import ResultSet +from boto.s3.acl import ACL, Policy, CannedACLStrings, Grant +from boto.s3.key import Key +from boto.s3.prefix import Prefix +from boto.s3.deletemarker import DeleteMarker +from boto.s3.user import User +from boto.s3.multipart import MultiPartUpload +from boto.s3.multipart import CompleteMultiPartUpload +from boto.s3.bucketlistresultset import BucketListResultSet +from boto.s3.bucketlistresultset import VersionedBucketListResultSet +from boto.s3.bucketlistresultset import MultiPartUploadListResultSet +import boto.jsonresponse +import boto.utils +import xml.sax +import urllib +import re +from collections import defaultdict + +# as per http://goo.gl/BDuud (02/19/2011) +class S3WebsiteEndpointTranslate: + trans_region = defaultdict(lambda :'s3-website-us-east-1') + + trans_region['EU'] = 's3-website-eu-west-1' + trans_region['us-west-1'] = 's3-website-us-west-1' + trans_region['ap-southeast-1'] = 's3-website-ap-southeast-1' + + @classmethod + def translate_region(self, reg): + return self.trans_region[reg] + +S3Permissions = ['READ', 'WRITE', 'READ_ACP', 'WRITE_ACP', 'FULL_CONTROL'] + +class Bucket(object): + + BucketLoggingBody = """ + + + %s + %s + + """ + + EmptyBucketLoggingBody = """ + + """ + + LoggingGroup = 'http://acs.amazonaws.com/groups/s3/LogDelivery' + + BucketPaymentBody = """ + + %s + """ + + VersioningBody = """ + + %s + %s + """ + + WebsiteBody = """ + + %s + %s + """ + + WebsiteErrorFragment = """%s""" + + VersionRE = '([A-Za-z]+)' + MFADeleteRE = '([A-Za-z]+)' + + def __init__(self, connection=None, name=None, key_class=Key): + self.name = name + self.connection = connection + self.key_class = key_class + + def __repr__(self): + return '' % self.name + + def __iter__(self): + return iter(BucketListResultSet(self)) + + def __contains__(self, key_name): + return not (self.get_key(key_name) is None) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Name': + self.name = value + elif name == 'CreationDate': + self.creation_date = value + else: + setattr(self, name, value) + + def set_key_class(self, key_class): + """ + Set the Key class associated with this bucket. By default, this + would be the boto.s3.key.Key class but if you want to subclass that + for some reason this allows you to associate your new class with a + bucket so that when you call bucket.new_key() or when you get a listing + of keys in the bucket you will get an instances of your key class + rather than the default. + + :type key_class: class + :param key_class: A subclass of Key that can be more specific + """ + self.key_class = key_class + + def lookup(self, key_name, headers=None): + """ + Deprecated: Please use get_key method. + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + return self.get_key(key_name, headers=headers) + + def get_key(self, key_name, headers=None, version_id=None): + """ + Check to see if a particular key exists within the bucket. This + method uses a HEAD request to check for the existance of the key. + Returns: An instance of a Key object or None + + :type key_name: string + :param key_name: The name of the key to retrieve + + :rtype: :class:`boto.s3.key.Key` + :returns: A Key object from this bucket. + """ + if version_id: + query_args = 'versionId=%s' % version_id + else: + query_args = None + response = self.connection.make_request('HEAD', self.name, key_name, + headers=headers, + query_args=query_args) + # Allow any success status (2xx) - for example this lets us + # support Range gets, which return status 206: + if response.status/100 == 2: + response.read() + k = self.key_class(self) + provider = self.connection.provider + k.metadata = boto.utils.get_aws_metadata(response.msg, provider) + k.etag = response.getheader('etag') + k.content_type = response.getheader('content-type') + k.content_encoding = response.getheader('content-encoding') + k.last_modified = response.getheader('last-modified') + k.size = int(response.getheader('content-length')) + k.cache_control = response.getheader('cache-control') + k.name = key_name + k.handle_version_headers(response) + return k + else: + if response.status == 404: + response.read() + return None + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, '') + + def list(self, prefix='', delimiter='', marker='', headers=None): + """ + List key objects within a bucket. This returns an instance of an + BucketListResultSet that automatically handles all of the result + paging, etc. from S3. You just need to keep iterating until + there are no more results. + + Called with no arguments, this will return an iterator object across + all keys within the bucket. + + The Key objects returned by the iterator are obtained by parsing + the results of a GET on the bucket, also known as the List Objects + request. The XML returned by this request contains only a subset + of the information about each key. Certain metadata fields such + as Content-Type and user metadata are not available in the XML. + Therefore, if you want these additional metadata fields you will + have to do a HEAD request on the Key in the bucket. + + :type prefix: string + :param prefix: allows you to limit the listing to a particular + prefix. For example, if you call the method with + prefix='/foo/' then the iterator will only cycle + through the keys that begin with the string '/foo/'. + + :type delimiter: string + :param delimiter: can be used in conjunction with the prefix + to allow you to organize and browse your keys + hierarchically. See: + http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + for more details. + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` + :return: an instance of a BucketListResultSet that handles paging, etc + """ + return BucketListResultSet(self, prefix, delimiter, marker, headers) + + def list_versions(self, prefix='', delimiter='', key_marker='', + version_id_marker='', headers=None): + """ + List version objects within a bucket. This returns an instance of an + VersionedBucketListResultSet that automatically handles all of the result + paging, etc. from S3. You just need to keep iterating until + there are no more results. + Called with no arguments, this will return an iterator object across + all keys within the bucket. + + :type prefix: string + :param prefix: allows you to limit the listing to a particular + prefix. For example, if you call the method with + prefix='/foo/' then the iterator will only cycle + through the keys that begin with the string '/foo/'. + + :type delimiter: string + :param delimiter: can be used in conjunction with the prefix + to allow you to organize and browse your keys + hierarchically. See: + http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ + for more details. + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` + :return: an instance of a BucketListResultSet that handles paging, etc + """ + return VersionedBucketListResultSet(self, prefix, delimiter, key_marker, + version_id_marker, headers) + + def list_multipart_uploads(self, key_marker='', + upload_id_marker='', + headers=None): + """ + List multipart upload objects within a bucket. This returns an + instance of an MultiPartUploadListResultSet that automatically + handles all of the result paging, etc. from S3. You just need + to keep iterating until there are no more results. + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :rtype: :class:`boto.s3.bucketlistresultset.BucketListResultSet` + :return: an instance of a BucketListResultSet that handles paging, etc + """ + return MultiPartUploadListResultSet(self, key_marker, + upload_id_marker, + headers) + + def _get_all(self, element_map, initial_query_string='', + headers=None, **params): + l = [] + for k,v in params.items(): + k = k.replace('_', '-') + if k == 'maxkeys': + k = 'max-keys' + if isinstance(v, unicode): + v = v.encode('utf-8') + if v is not None and v != '': + l.append('%s=%s' % (urllib.quote(k), urllib.quote(str(v)))) + if len(l): + s = initial_query_string + '&' + '&'.join(l) + else: + s = initial_query_string + response = self.connection.make_request('GET', self.name, + headers=headers, query_args=s) + body = response.read() + boto.log.debug(body) + if response.status == 200: + rs = ResultSet(element_map) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_all_keys(self, headers=None, **params): + """ + A lower-level method for listing contents of a bucket. + This closely models the actual S3 API and requires you to manually + handle the paging of results. For a higher-level method + that handles the details of paging for you, you can use the list method. + + :type max_keys: int + :param max_keys: The maximum number of keys to retrieve + + :type prefix: string + :param prefix: The prefix of the keys you want to retrieve + + :type marker: string + :param marker: The "marker" of where you are in the result set + + :type delimiter: string + :param delimiter: If this optional, Unicode string parameter + is included with your request, then keys that + contain the same string between the prefix and + the first occurrence of the delimiter will be + rolled up into a single result element in the + CommonPrefixes collection. These rolled-up keys + are not returned elsewhere in the response. + + :rtype: ResultSet + :return: The result from S3 listing the keys requested + + """ + return self._get_all([('Contents', self.key_class), + ('CommonPrefixes', Prefix)], + '', headers, **params) + + def get_all_versions(self, headers=None, **params): + """ + A lower-level, version-aware method for listing contents of a bucket. + This closely models the actual S3 API and requires you to manually + handle the paging of results. For a higher-level method + that handles the details of paging for you, you can use the list method. + + :type max_keys: int + :param max_keys: The maximum number of keys to retrieve + + :type prefix: string + :param prefix: The prefix of the keys you want to retrieve + + :type key_marker: string + :param key_marker: The "marker" of where you are in the result set + with respect to keys. + + :type version_id_marker: string + :param version_id_marker: The "marker" of where you are in the result + set with respect to version-id's. + + :type delimiter: string + :param delimiter: If this optional, Unicode string parameter + is included with your request, then keys that + contain the same string between the prefix and + the first occurrence of the delimiter will be + rolled up into a single result element in the + CommonPrefixes collection. These rolled-up keys + are not returned elsewhere in the response. + + :rtype: ResultSet + :return: The result from S3 listing the keys requested + + """ + return self._get_all([('Version', self.key_class), + ('CommonPrefixes', Prefix), + ('DeleteMarker', DeleteMarker)], + 'versions', headers, **params) + + def get_all_multipart_uploads(self, headers=None, **params): + """ + A lower-level, version-aware method for listing active + MultiPart uploads for a bucket. This closely models the + actual S3 API and requires you to manually handle the paging + of results. For a higher-level method that handles the + details of paging for you, you can use the list method. + + :type max_uploads: int + :param max_uploads: The maximum number of uploads to retrieve. + Default value is 1000. + + :type key_marker: string + :param key_marker: Together with upload_id_marker, this parameter + specifies the multipart upload after which listing + should begin. If upload_id_marker is not specified, + only the keys lexicographically greater than the + specified key_marker will be included in the list. + + If upload_id_marker is specified, any multipart + uploads for a key equal to the key_marker might + also be included, provided those multipart uploads + have upload IDs lexicographically greater than the + specified upload_id_marker. + + :type upload_id_marker: string + :param upload_id_marker: Together with key-marker, specifies + the multipart upload after which listing + should begin. If key_marker is not specified, + the upload_id_marker parameter is ignored. + Otherwise, any multipart uploads for a key + equal to the key_marker might be included + in the list only if they have an upload ID + lexicographically greater than the specified + upload_id_marker. + + + :rtype: ResultSet + :return: The result from S3 listing the uploads requested + + """ + return self._get_all([('Upload', MultiPartUpload)], + 'uploads', headers, **params) + + def new_key(self, key_name=None): + """ + Creates a new key + + :type key_name: string + :param key_name: The name of the key to create + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + return self.key_class(self, key_name) + + def generate_url(self, expires_in, method='GET', + headers=None, force_http=False): + return self.connection.generate_url(expires_in, method, self.name, + headers=headers, + force_http=force_http) + + def delete_key(self, key_name, headers=None, + version_id=None, mfa_token=None): + """ + Deletes a key from the bucket. If a version_id is provided, + only that version of the key will be deleted. + + :type key_name: string + :param key_name: The key name to delete + + :type version_id: string + :param version_id: The version ID (optional) + + :type mfa_token: tuple or list of strings + :param mfa_token: A tuple or list consisting of the serial number + from the MFA device and the current value of + the six-digit token associated with the device. + This value is required anytime you are + deleting versioned objects from a bucket + that has the MFADelete option on the bucket. + """ + provider = self.connection.provider + if version_id: + query_args = 'versionId=%s' % version_id + else: + query_args = None + if mfa_token: + if not headers: + headers = {} + headers[provider.mfa_header] = ' '.join(mfa_token) + response = self.connection.make_request('DELETE', self.name, key_name, + headers=headers, + query_args=query_args) + body = response.read() + if response.status != 204: + raise provider.storage_response_error(response.status, + response.reason, body) + + def copy_key(self, new_key_name, src_bucket_name, + src_key_name, metadata=None, src_version_id=None, + storage_class='STANDARD', preserve_acl=False): + """ + Create a new key in the bucket by copying another existing key. + + :type new_key_name: string + :param new_key_name: The name of the new key + + :type src_bucket_name: string + :param src_bucket_name: The name of the source bucket + + :type src_key_name: string + :param src_key_name: The name of the source key + + :type src_version_id: string + :param src_version_id: The version id for the key. This param + is optional. If not specified, the newest + version of the key will be copied. + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :type storage_class: string + :param storage_class: The storage class of the new key. + By default, the new key will use the + standard storage class. Possible values are: + STANDARD | REDUCED_REDUNDANCY + + :type preserve_acl: bool + :param preserve_acl: If True, the ACL from the source key + will be copied to the destination + key. If False, the destination key + will have the default ACL. + Note that preserving the ACL in the + new key object will require two + additional API calls to S3, one to + retrieve the current ACL and one to + set that ACL on the new object. If + you don't care about the ACL, a value + of False will be significantly more + efficient. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + if preserve_acl: + acl = self.get_xml_acl(src_key_name) + src = '%s/%s' % (src_bucket_name, urllib.quote(src_key_name)) + if src_version_id: + src += '?version_id=%s' % src_version_id + provider = self.connection.provider + headers = {provider.copy_source_header : src} + if storage_class != 'STANDARD': + headers[provider.storage_class_header] = storage_class + if metadata: + headers[provider.metadata_directive_header] = 'REPLACE' + headers = boto.utils.merge_meta(headers, metadata) + else: + headers[provider.metadata_directive_header] = 'COPY' + response = self.connection.make_request('PUT', self.name, new_key_name, + headers=headers) + body = response.read() + if response.status == 200: + key = self.new_key(new_key_name) + h = handler.XmlHandler(key, self) + xml.sax.parseString(body, h) + if hasattr(key, 'Error'): + raise provider.storage_copy_error(key.Code, key.Message, body) + key.handle_version_headers(response) + if preserve_acl: + self.set_xml_acl(acl, new_key_name) + return key + else: + raise provider.storage_response_error(response.status, response.reason, body) + + def set_canned_acl(self, acl_str, key_name='', headers=None, + version_id=None): + assert acl_str in CannedACLStrings + + if headers: + headers[self.connection.provider.acl_header] = acl_str + else: + headers={self.connection.provider.acl_header: acl_str} + + query_args='acl' + if version_id: + query_args += '&versionId=%s' % version_id + response = self.connection.make_request('PUT', self.name, key_name, + headers=headers, query_args=query_args) + body = response.read() + if response.status != 200: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_xml_acl(self, key_name='', headers=None, version_id=None): + query_args = 'acl' + if version_id: + query_args += '&versionId=%s' % version_id + response = self.connection.make_request('GET', self.name, key_name, + query_args=query_args, + headers=headers) + body = response.read() + if response.status != 200: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + return body + + def set_xml_acl(self, acl_str, key_name='', headers=None, version_id=None): + query_args = 'acl' + if version_id: + query_args += '&versionId=%s' % version_id + response = self.connection.make_request('PUT', self.name, key_name, + data=acl_str, + query_args=query_args, + headers=headers) + body = response.read() + if response.status != 200: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def set_acl(self, acl_or_str, key_name='', headers=None, version_id=None): + if isinstance(acl_or_str, Policy): + self.set_xml_acl(acl_or_str.to_xml(), key_name, + headers, version_id) + else: + self.set_canned_acl(acl_or_str, key_name, + headers, version_id) + + def get_acl(self, key_name='', headers=None, version_id=None): + query_args = 'acl' + if version_id: + query_args += '&versionId=%s' % version_id + response = self.connection.make_request('GET', self.name, key_name, + query_args=query_args, + headers=headers) + body = response.read() + if response.status == 200: + policy = Policy(self) + h = handler.XmlHandler(policy, self) + xml.sax.parseString(body, h) + return policy + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def make_public(self, recursive=False, headers=None): + self.set_canned_acl('public-read', headers=headers) + if recursive: + for key in self: + self.set_canned_acl('public-read', key.name, headers=headers) + + def add_email_grant(self, permission, email_address, + recursive=False, headers=None): + """ + Convenience method that provides a quick way to add an email grant + to a bucket. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL + and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + + :type email_address: string + :param email_address: The email address associated with the AWS + account your are granting the permission to. + + :type recursive: boolean + :param recursive: A boolean value to controls whether the command + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + if permission not in S3Permissions: + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_email_grant(permission, email_address, headers=headers) + + def add_user_grant(self, permission, user_id, + recursive=False, headers=None): + """ + Convenience method that provides a quick way to add a canonical + user grant to a bucket. This method retrieves the current ACL, + creates a new grant based on the parameters passed in, adds that + grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + + :type user_id: string + :param user_id: The canonical user id associated with the AWS + account your are granting the permission to. + + :type recursive: boolean + :param recursive: A boolean value to controls whether the command + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + if permission not in S3Permissions: + raise self.connection.provider.storage_permissions_error( + 'Unknown Permission: %s' % permission) + policy = self.get_acl(headers=headers) + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy, headers=headers) + if recursive: + for key in self: + key.add_user_grant(permission, user_id, headers=headers) + + def list_grants(self, headers=None): + policy = self.get_acl(headers=headers) + return policy.acl.grants + + def get_location(self): + """ + Returns the LocationConstraint for the bucket. + + :rtype: str + :return: The LocationConstraint for the bucket or the empty + string if no constraint was specified when bucket + was created. + """ + response = self.connection.make_request('GET', self.name, + query_args='location') + body = response.read() + if response.status == 200: + rs = ResultSet(self) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs.LocationConstraint + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def enable_logging(self, target_bucket, target_prefix='', headers=None): + if isinstance(target_bucket, Bucket): + target_bucket = target_bucket.name + body = self.BucketLoggingBody % (target_bucket, target_prefix) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def disable_logging(self, headers=None): + body = self.EmptyBucketLoggingBody + response = self.connection.make_request('PUT', self.name, data=body, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_logging_status(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='logging', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def set_as_logging_target(self, headers=None): + policy = self.get_acl(headers=headers) + g1 = Grant(permission='WRITE', type='Group', uri=self.LoggingGroup) + g2 = Grant(permission='READ_ACP', type='Group', uri=self.LoggingGroup) + policy.acl.add_grant(g1) + policy.acl.add_grant(g2) + self.set_acl(policy, headers=headers) + + def get_request_payment(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def set_request_payment(self, payer='BucketOwner', headers=None): + body = self.BucketPaymentBody % payer + response = self.connection.make_request('PUT', self.name, data=body, + query_args='requestPayment', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def configure_versioning(self, versioning, mfa_delete=False, + mfa_token=None, headers=None): + """ + Configure versioning for this bucket. + Note: This feature is currently in beta release and is available + only in the Northern California region. + + :type versioning: bool + :param versioning: A boolean indicating whether version is + enabled (True) or disabled (False). + + :type mfa_delete: bool + :param mfa_delete: A boolean indicating whether the Multi-Factor + Authentication Delete feature is enabled (True) + or disabled (False). If mfa_delete is enabled + then all Delete operations will require the + token from your MFA device to be passed in + the request. + + :type mfa_token: tuple or list of strings + :param mfa_token: A tuple or list consisting of the serial number + from the MFA device and the current value of + the six-digit token associated with the device. + This value is required when you are changing + the status of the MfaDelete property of + the bucket. + """ + if versioning: + ver = 'Enabled' + else: + ver = 'Suspended' + if mfa_delete: + mfa = 'Enabled' + else: + mfa = 'Disabled' + body = self.VersioningBody % (ver, mfa) + if mfa_token: + if not headers: + headers = {} + provider = self.connection.provider + headers[provider.mfa_header] = ' '.join(mfa_token) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='versioning', headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_versioning_status(self, headers=None): + """ + Returns the current status of versioning on the bucket. + + :rtype: dict + :returns: A dictionary containing a key named 'Versioning' + that can have a value of either Enabled, Disabled, + or Suspended. Also, if MFADelete has ever been enabled + on the bucket, the dictionary will contain a key + named 'MFADelete' which will have a value of either + Enabled or Suspended. + """ + response = self.connection.make_request('GET', self.name, + query_args='versioning', headers=headers) + body = response.read() + boto.log.debug(body) + if response.status == 200: + d = {} + ver = re.search(self.VersionRE, body) + if ver: + d['Versioning'] = ver.group(1) + mfa = re.search(self.MFADeleteRE, body) + if mfa: + d['MfaDelete'] = mfa.group(1) + return d + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def configure_website(self, suffix, error_key='', headers=None): + """ + Configure this bucket to act as a website + + :type suffix: str + :param suffix: Suffix that is appended to a request that is for a + "directory" on the website endpoint (e.g. if the suffix + is index.html and you make a request to + samplebucket/images/ the data that is returned will + be for the object with the key name images/index.html). + The suffix must not be empty and must not include a + slash character. + + :type error_key: str + :param error_key: The object key name to use when a 4XX class + error occurs. This is optional. + + """ + if error_key: + error_frag = self.WebsiteErrorFragment % error_key + else: + error_frag = '' + body = self.WebsiteBody % (suffix, error_frag) + response = self.connection.make_request('PUT', self.name, data=body, + query_args='website', + headers=headers) + body = response.read() + if response.status == 200: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_website_configuration(self, headers=None): + """ + Returns the current status of website configuration on the bucket. + + :rtype: dict + :returns: A dictionary containing a Python representation + of the XML response from S3. The overall structure is: + + * WebsiteConfiguration + * IndexDocument + * Suffix : suffix that is appended to request that + is for a "directory" on the website endpoint + * ErrorDocument + * Key : name of object to serve when an error occurs + """ + response = self.connection.make_request('GET', self.name, + query_args='website', headers=headers) + body = response.read() + boto.log.debug(body) + if response.status == 200: + e = boto.jsonresponse.Element() + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def delete_website_configuration(self, headers=None): + """ + Removes all website configuration from the bucket. + """ + response = self.connection.make_request('DELETE', self.name, + query_args='website', headers=headers) + body = response.read() + boto.log.debug(body) + if response.status == 204: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def get_website_endpoint(self): + """ + Returns the fully qualified hostname to use is you want to access this + bucket as a website. This doesn't validate whether the bucket has + been correctly configured as a website or not. + """ + l = [self.name] + l.append(S3WebsiteEndpointTranslate.translate_region(self.get_location())) + l.append('.'.join(self.connection.host.split('.')[-2:])) + return '.'.join(l) + + def get_policy(self, headers=None): + response = self.connection.make_request('GET', self.name, + query_args='policy', headers=headers) + body = response.read() + if response.status == 200: + return body + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def set_policy(self, policy, headers=None): + response = self.connection.make_request('PUT', self.name, + data=policy, + query_args='policy', + headers=headers) + body = response.read() + if response.status >= 200 and response.status <= 204: + return True + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def initiate_multipart_upload(self, key_name, headers=None, reduced_redundancy=False): + """ + Start a multipart upload operation. + + :type key_name: string + :param key_name: The name of the key that will ultimately result from + this multipart upload operation. This will be exactly + as the key appears in the bucket after the upload + process has been completed. + + :type headers: dict + :param headers: Additional HTTP headers to send and store with the + resulting key in S3. + + :type reduced_redundancy: boolean + :param reduced_redundancy: In multipart uploads, the storage class is + specified when initiating the upload, + not when uploading individual parts. So + if you want the resulting key to use the + reduced redundancy storage class set this + flag when you initiate the upload. + """ + query_args = 'uploads' + if headers is None: + headers = {} + if reduced_redundancy: + storage_class_header = self.connection.provider.storage_class_header + if storage_class_header: + headers[storage_class_header] = 'REDUCED_REDUNDANCY' + # TODO: what if the provider doesn't support reduced redundancy? + # (see boto.s3.key.Key.set_contents_from_file) + response = self.connection.make_request('POST', self.name, key_name, + query_args=query_args, + headers=headers) + body = response.read() + boto.log.debug(body) + if response.status == 200: + resp = MultiPartUpload(self) + h = handler.XmlHandler(resp, self) + xml.sax.parseString(body, h) + return resp + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def complete_multipart_upload(self, key_name, upload_id, + xml_body, headers=None): + """ + Complete a multipart upload operation. + """ + query_args = 'uploadId=%s' % upload_id + if headers is None: + headers = {} + headers['Content-Type'] = 'text/xml' + response = self.connection.make_request('POST', self.name, key_name, + query_args=query_args, + headers=headers, data=xml_body) + contains_error = False + body = response.read() + # Some errors will be reported in the body of the response + # even though the HTTP response code is 200. This check + # does a quick and dirty peek in the body for an error element. + if body.find('') > 0: + contains_error = True + boto.log.debug(body) + if response.status == 200 and not contains_error: + resp = CompleteMultiPartUpload(self) + h = handler.XmlHandler(resp, self) + xml.sax.parseString(body, h) + return resp + else: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def cancel_multipart_upload(self, key_name, upload_id, headers=None): + query_args = 'uploadId=%s' % upload_id + response = self.connection.make_request('DELETE', self.name, key_name, + query_args=query_args, + headers=headers) + body = response.read() + boto.log.debug(body) + if response.status != 204: + raise self.connection.provider.storage_response_error( + response.status, response.reason, body) + + def delete(self, headers=None): + return self.connection.delete_bucket(self.name, headers=headers) + diff --git a/backup/src/boto/s3/bucketlistresultset.py b/backup/src/boto/s3/bucketlistresultset.py new file mode 100644 index 0000000..0123663 --- /dev/null +++ b/backup/src/boto/s3/bucketlistresultset.py @@ -0,0 +1,139 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +def bucket_lister(bucket, prefix='', delimiter='', marker='', headers=None): + """ + A generator function for listing keys in a bucket. + """ + more_results = True + k = None + while more_results: + rs = bucket.get_all_keys(prefix=prefix, marker=marker, + delimiter=delimiter, headers=headers) + for k in rs: + yield k + if k: + marker = k.name + more_results= rs.is_truncated + +class BucketListResultSet: + """ + A resultset for listing keys within a bucket. Uses the bucket_lister + generator function and implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of keys within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + + def __init__(self, bucket=None, prefix='', delimiter='', marker='', headers=None): + self.bucket = bucket + self.prefix = prefix + self.delimiter = delimiter + self.marker = marker + self.headers = headers + + def __iter__(self): + return bucket_lister(self.bucket, prefix=self.prefix, + delimiter=self.delimiter, marker=self.marker, headers=self.headers) + +def versioned_bucket_lister(bucket, prefix='', delimiter='', + key_marker='', version_id_marker='', headers=None): + """ + A generator function for listing versions in a bucket. + """ + more_results = True + k = None + while more_results: + rs = bucket.get_all_versions(prefix=prefix, key_marker=key_marker, + version_id_marker=version_id_marker, + delimiter=delimiter, headers=headers) + for k in rs: + yield k + key_marker = rs.next_key_marker + version_id_marker = rs.next_version_id_marker + more_results= rs.is_truncated + +class VersionedBucketListResultSet: + """ + A resultset for listing versions within a bucket. Uses the bucket_lister + generator function and implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of keys within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + + def __init__(self, bucket=None, prefix='', delimiter='', key_marker='', + version_id_marker='', headers=None): + self.bucket = bucket + self.prefix = prefix + self.delimiter = delimiter + self.key_marker = key_marker + self.version_id_marker = version_id_marker + self.headers = headers + + def __iter__(self): + return versioned_bucket_lister(self.bucket, prefix=self.prefix, + delimiter=self.delimiter, + key_marker=self.key_marker, + version_id_marker=self.version_id_marker, + headers=self.headers) + +def multipart_upload_lister(bucket, key_marker='', + upload_id_marker='', + headers=None): + """ + A generator function for listing multipart uploads in a bucket. + """ + more_results = True + k = None + while more_results: + rs = bucket.get_all_multipart_uploads(key_marker=key_marker, + upload_id_marker=upload_id_marker, + headers=headers) + for k in rs: + yield k + key_marker = rs.next_key_marker + upload_id_marker = rs.next_upload_id_marker + more_results= rs.is_truncated + +class MultiPartUploadListResultSet: + """ + A resultset for listing multipart uploads within a bucket. + Uses the multipart_upload_lister generator function and + implements the iterator interface. This + transparently handles the results paging from S3 so even if you have + many thousands of uploads within the bucket you can iterate over all + keys in a reasonably efficient manner. + """ + def __init__(self, bucket=None, key_marker='', + upload_id_marker='', headers=None): + self.bucket = bucket + self.key_marker = key_marker + self.upload_id_marker = upload_id_marker + self.headers = headers + + def __iter__(self): + return multipart_upload_lister(self.bucket, + key_marker=self.key_marker, + upload_id_marker=self.upload_id_marker, + headers=self.headers) + + diff --git a/backup/src/boto/s3/connection.py b/backup/src/boto/s3/connection.py new file mode 100644 index 0000000..25ba4ab --- /dev/null +++ b/backup/src/boto/s3/connection.py @@ -0,0 +1,401 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import urllib, base64 +import time +import boto.utils +from boto.connection import AWSAuthConnection +from boto import handler +from boto.provider import Provider +from boto.s3.bucket import Bucket +from boto.s3.key import Key +from boto.resultset import ResultSet +from boto.exception import BotoClientError + +def check_lowercase_bucketname(n): + """ + Bucket names must not contain uppercase characters. We check for + this by appending a lowercase character and testing with islower(). + Note this also covers cases like numeric bucket names with dashes. + + >>> check_lowercase_bucketname("Aaaa") + Traceback (most recent call last): + ... + BotoClientError: S3Error: Bucket names cannot contain upper-case + characters when using either the sub-domain or virtual hosting calling + format. + + >>> check_lowercase_bucketname("1234-5678-9123") + True + >>> check_lowercase_bucketname("abcdefg1234") + True + """ + if not (n + 'a').islower(): + raise BotoClientError("Bucket names cannot contain upper-case " \ + "characters when using either the sub-domain or virtual " \ + "hosting calling format.") + return True + +def assert_case_insensitive(f): + def wrapper(*args, **kwargs): + if len(args) == 3 and check_lowercase_bucketname(args[2]): + pass + return f(*args, **kwargs) + return wrapper + +class _CallingFormat: + + def build_url_base(self, connection, protocol, server, bucket, key=''): + url_base = '%s://' % protocol + url_base += self.build_host(server, bucket) + url_base += connection.get_path(self.build_path_base(bucket, key)) + return url_base + + def build_host(self, server, bucket): + if bucket == '': + return server + else: + return self.get_bucket_server(server, bucket) + + def build_auth_path(self, bucket, key=''): + path = '' + if bucket != '': + path = '/' + bucket + return path + '/%s' % urllib.quote(key) + + def build_path_base(self, bucket, key=''): + return '/%s' % urllib.quote(key) + +class SubdomainCallingFormat(_CallingFormat): + + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return '%s.%s' % (bucket, server) + +class VHostCallingFormat(_CallingFormat): + + @assert_case_insensitive + def get_bucket_server(self, server, bucket): + return bucket + +class OrdinaryCallingFormat(_CallingFormat): + + def get_bucket_server(self, server, bucket): + return server + + def build_path_base(self, bucket, key=''): + path_base = '/' + if bucket: + path_base += "%s/" % bucket + return path_base + urllib.quote(key) + +class Location: + DEFAULT = '' # US Classic Region + EU = 'EU' + USWest = 'us-west-1' + APSoutheast = 'ap-southeast-1' + +class S3Connection(AWSAuthConnection): + + DefaultHost = 's3.amazonaws.com' + QueryString = 'Signature=%s&Expires=%d&AWSAccessKeyId=%s' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, + host=DefaultHost, debug=0, https_connection_factory=None, + calling_format=SubdomainCallingFormat(), path='/', provider='aws', + bucket_class=Bucket): + self.calling_format = calling_format + self.bucket_class = bucket_class + AWSAuthConnection.__init__(self, host, + aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + debug=debug, https_connection_factory=https_connection_factory, + path=path, provider=provider) + + def _required_auth_capability(self): + return ['s3'] + + def __iter__(self): + for bucket in self.get_all_buckets(): + yield bucket + + def __contains__(self, bucket_name): + return not (self.lookup(bucket_name) is None) + + def set_bucket_class(self, bucket_class): + """ + Set the Bucket class associated with this bucket. By default, this + would be the boto.s3.key.Bucket class but if you want to subclass that + for some reason this allows you to associate your new class. + + :type bucket_class: class + :param bucket_class: A subclass of Bucket that can be more specific + """ + self.bucket_class = bucket_class + + def build_post_policy(self, expiration_time, conditions): + """ + Taken from the AWS book Python examples and modified for use with boto + """ + assert type(expiration_time) == time.struct_time, \ + 'Policy document must include a valid expiration Time object' + + # Convert conditions object mappings to condition statements + + return '{"expiration": "%s",\n"conditions": [%s]}' % \ + (time.strftime(boto.utils.ISO8601, expiration_time), ",".join(conditions)) + + + def build_post_form_args(self, bucket_name, key, expires_in = 6000, + acl = None, success_action_redirect = None, max_content_length = None, + http_method = "http", fields=None, conditions=None): + """ + Taken from the AWS book Python examples and modified for use with boto + This only returns the arguments required for the post form, not the actual form + This does not return the file input field which also needs to be added + + :param bucket_name: Bucket to submit to + :type bucket_name: string + + :param key: Key name, optionally add ${filename} to the end to attach the submitted filename + :type key: string + + :param expires_in: Time (in seconds) before this expires, defaults to 6000 + :type expires_in: integer + + :param acl: ACL rule to use, if any + :type acl: :class:`boto.s3.acl.ACL` + + :param success_action_redirect: URL to redirect to on success + :type success_action_redirect: string + + :param max_content_length: Maximum size for this file + :type max_content_length: integer + + :type http_method: string + :param http_method: HTTP Method to use, "http" or "https" + + + :rtype: dict + :return: A dictionary containing field names/values as well as a url to POST to + + .. code-block:: python + + { + "action": action_url_to_post_to, + "fields": [ + { + "name": field_name, + "value": field_value + }, + { + "name": field_name2, + "value": field_value2 + } + ] + } + + """ + if fields == None: + fields = [] + if conditions == None: + conditions = [] + expiration = time.gmtime(int(time.time() + expires_in)) + + # Generate policy document + conditions.append('{"bucket": "%s"}' % bucket_name) + if key.endswith("${filename}"): + conditions.append('["starts-with", "$key", "%s"]' % key[:-len("${filename}")]) + else: + conditions.append('{"key": "%s"}' % key) + if acl: + conditions.append('{"acl": "%s"}' % acl) + fields.append({ "name": "acl", "value": acl}) + if success_action_redirect: + conditions.append('{"success_action_redirect": "%s"}' % success_action_redirect) + fields.append({ "name": "success_action_redirect", "value": success_action_redirect}) + if max_content_length: + conditions.append('["content-length-range", 0, %i]' % max_content_length) + fields.append({"name":'content-length-range', "value": "0,%i" % max_content_length}) + + policy = self.build_post_policy(expiration, conditions) + + # Add the base64-encoded policy document as the 'policy' field + policy_b64 = base64.b64encode(policy) + fields.append({"name": "policy", "value": policy_b64}) + + # Add the AWS access key as the 'AWSAccessKeyId' field + fields.append({"name": "AWSAccessKeyId", "value": self.aws_access_key_id}) + + # Add signature for encoded policy document as the 'AWSAccessKeyId' field + signature = self._auth_handler.sign_string(policy_b64) + fields.append({"name": "signature", "value": signature}) + fields.append({"name": "key", "value": key}) + + # HTTPS protocol will be used if the secure HTTP option is enabled. + url = '%s://%s/' % (http_method, self.calling_format.build_host(self.server_name(), bucket_name)) + + return {"action": url, "fields": fields} + + + def generate_url(self, expires_in, method, bucket='', key='', + headers=None, query_auth=True, force_http=False): + if not headers: + headers = {} + expires = int(time.time() + expires_in) + auth_path = self.calling_format.build_auth_path(bucket, key) + auth_path = self.get_path(auth_path) + c_string = boto.utils.canonical_string(method, auth_path, headers, + expires, self.provider) + b64_hmac = self._auth_handler.sign_string(c_string) + encoded_canonical = urllib.quote_plus(b64_hmac) + self.calling_format.build_path_base(bucket, key) + if query_auth: + query_part = '?' + self.QueryString % (encoded_canonical, expires, + self.aws_access_key_id) + sec_hdr = self.provider.security_token_header + if sec_hdr in headers: + query_part += ('&%s=%s' % (sec_hdr, + urllib.quote(headers[sec_hdr]))); + else: + query_part = '' + if force_http: + protocol = 'http' + port = 80 + else: + protocol = self.protocol + port = self.port + return self.calling_format.build_url_base(self, protocol, self.server_name(port), + bucket, key) + query_part + + def get_all_buckets(self, headers=None): + response = self.make_request('GET', headers=headers) + body = response.read() + if response.status > 300: + raise self.provider.storage_response_error( + response.status, response.reason, body) + rs = ResultSet([('Bucket', self.bucket_class)]) + h = handler.XmlHandler(rs, self) + xml.sax.parseString(body, h) + return rs + + def get_canonical_user_id(self, headers=None): + """ + Convenience method that returns the "CanonicalUserID" of the user who's credentials + are associated with the connection. The only way to get this value is to do a GET + request on the service which returns all buckets associated with the account. As part + of that response, the canonical userid is returned. This method simply does all of + that and then returns just the user id. + + :rtype: string + :return: A string containing the canonical user id. + """ + rs = self.get_all_buckets(headers=headers) + return rs.ID + + def get_bucket(self, bucket_name, validate=True, headers=None): + bucket = self.bucket_class(self, bucket_name) + if validate: + bucket.get_all_keys(headers, maxkeys=0) + return bucket + + def lookup(self, bucket_name, validate=True, headers=None): + try: + bucket = self.get_bucket(bucket_name, validate, headers=headers) + except: + bucket = None + return bucket + + def create_bucket(self, bucket_name, headers=None, + location=Location.DEFAULT, policy=None): + """ + Creates a new located bucket. By default it's in the USA. You can pass + Location.EU to create an European bucket. + + :type bucket_name: string + :param bucket_name: The name of the new bucket + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type location: :class:`boto.s3.connection.Location` + :param location: The location of the new bucket + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + """ + check_lowercase_bucketname(bucket_name) + + if policy: + if headers: + headers[self.provider.acl_header] = policy + else: + headers = {self.provider.acl_header : policy} + if location == Location.DEFAULT: + data = '' + else: + data = '' + \ + location + '' + response = self.make_request('PUT', bucket_name, headers=headers, + data=data) + body = response.read() + if response.status == 409: + raise self.provider.storage_create_error( + response.status, response.reason, body) + if response.status == 200: + return self.bucket_class(self, bucket_name) + else: + raise self.provider.storage_response_error( + response.status, response.reason, body) + + def delete_bucket(self, bucket, headers=None): + response = self.make_request('DELETE', bucket, headers=headers) + body = response.read() + if response.status != 204: + raise self.provider.storage_response_error( + response.status, response.reason, body) + + def make_request(self, method, bucket='', key='', headers=None, data='', + query_args=None, sender=None, override_num_retries=None): + if isinstance(bucket, self.bucket_class): + bucket = bucket.name + if isinstance(key, Key): + key = key.name + path = self.calling_format.build_path_base(bucket, key) + boto.log.debug('path=%s' % path) + auth_path = self.calling_format.build_auth_path(bucket, key) + boto.log.debug('auth_path=%s' % auth_path) + host = self.calling_format.build_host(self.server_name(), bucket) + if query_args: + path += '?' + query_args + boto.log.debug('path=%s' % path) + auth_path += '?' + query_args + boto.log.debug('auth_path=%s' % auth_path) + return AWSAuthConnection.make_request(self, method, path, headers, + data, host, auth_path, sender, + override_num_retries=override_num_retries) + diff --git a/backup/src/boto/s3/deletemarker.py b/backup/src/boto/s3/deletemarker.py new file mode 100644 index 0000000..3462d42 --- /dev/null +++ b/backup/src/boto/s3/deletemarker.py @@ -0,0 +1,56 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.s3.user import User + +class DeleteMarker: + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + self.is_latest = False + self.last_modified = None + self.owner = None + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + else: + return None + + def endElement(self, name, value, connection): + if name == 'Key': + self.name = value.encode('utf-8') + elif name == 'IsLatest': + if value == 'true': + self.is_lastest = True + else: + self.is_latest = False + elif name == 'LastModified': + self.last_modified = value + elif name == 'Owner': + pass + elif name == 'VersionId': + self.version_id = value + else: + setattr(self, name, value) + + diff --git a/backup/src/boto/s3/key.py b/backup/src/boto/s3/key.py new file mode 100644 index 0000000..c7e77f4 --- /dev/null +++ b/backup/src/boto/s3/key.py @@ -0,0 +1,1059 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import mimetypes +import os +import rfc822 +import StringIO +import base64 +import boto.utils +from boto.exception import BotoClientError +from boto.provider import Provider +from boto.s3.user import User +from boto import UserAgent +try: + from hashlib import md5 +except ImportError: + from md5 import md5 + + +class Key(object): + + DefaultContentType = 'application/octet-stream' + + BufferSize = 8192 + + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + self.metadata = {} + self.cache_control = None + self.content_type = self.DefaultContentType + self.content_encoding = None + self.filename = None + self.etag = None + self.last_modified = None + self.owner = None + self.storage_class = 'STANDARD' + self.md5 = None + self.base64md5 = None + self.path = None + self.resp = None + self.mode = None + self.size = None + self.version_id = None + self.source_version_id = None + self.delete_marker = False + + def __repr__(self): + if self.bucket: + return '' % (self.bucket.name, self.name) + else: + return '' % self.name + + def __getattr__(self, name): + if name == 'key': + return self.name + else: + raise AttributeError + + def __setattr__(self, name, value): + if name == 'key': + self.__dict__['name'] = value + else: + self.__dict__[name] = value + + def __iter__(self): + return self + + @property + def provider(self): + provider = None + if self.bucket: + if self.bucket.connection: + provider = self.bucket.connection.provider + return provider + + def get_md5_from_hexdigest(self, md5_hexdigest): + """ + A utility function to create the 2-tuple (md5hexdigest, base64md5) + from just having a precalculated md5_hexdigest. + """ + import binascii + digest = binascii.unhexlify(md5_hexdigest) + base64md5 = base64.encodestring(digest) + if base64md5[-1] == '\n': + base64md5 = base64md5[0:-1] + return (md5_hexdigest, base64md5) + + def handle_version_headers(self, resp, force=False): + provider = self.bucket.connection.provider + # If the Key object already has a version_id attribute value, it + # means that it represents an explicit version and the user is + # doing a get_contents_*(version_id=) to retrieve another + # version of the Key. In that case, we don't really want to + # overwrite the version_id in this Key object. Comprende? + if self.version_id is None or force: + self.version_id = resp.getheader(provider.version_id, None) + self.source_version_id = resp.getheader(provider.copy_source_version_id, None) + if resp.getheader(provider.delete_marker, 'false') == 'true': + self.delete_marker = True + else: + self.delete_marker = False + + def open_read(self, headers=None, query_args=None, + override_num_retries=None, response_headers=None): + """ + Open this key for reading + + :type headers: dict + :param headers: Headers to pass in the web request + + :type query_args: string + :param query_args: Arguments to pass in the query string (ie, 'torrent') + + :type override_num_retries: int + :param override_num_retries: If not None will override configured + num_retries parameter for underlying GET. + + :type response_headers: dict + :param response_headers: A dictionary containing HTTP headers/values + that will override any headers associated with + the stored object in the response. + See http://goo.gl/EWOPb for details. + """ + if self.resp == None: + self.mode = 'r' + + provider = self.bucket.connection.provider + self.resp = self.bucket.connection.make_request( + 'GET', self.bucket.name, self.name, headers, + query_args=query_args, + override_num_retries=override_num_retries) + if self.resp.status < 199 or self.resp.status > 299: + body = self.resp.read() + raise provider.storage_response_error(self.resp.status, + self.resp.reason, body) + response_headers = self.resp.msg + self.metadata = boto.utils.get_aws_metadata(response_headers, + provider) + for name,value in response_headers.items(): + if name.lower() == 'content-length': + self.size = int(value) + elif name.lower() == 'etag': + self.etag = value + elif name.lower() == 'content-type': + self.content_type = value + elif name.lower() == 'content-encoding': + self.content_encoding = value + elif name.lower() == 'last-modified': + self.last_modified = value + elif name.lower() == 'cache-control': + self.cache_control = value + self.handle_version_headers(self.resp) + + def open_write(self, headers=None, override_num_retries=None): + """ + Open this key for writing. + Not yet implemented + + :type headers: dict + :param headers: Headers to pass in the write request + + :type override_num_retries: int + :param override_num_retries: If not None will override configured + num_retries parameter for underlying PUT. + """ + raise BotoClientError('Not Implemented') + + def open(self, mode='r', headers=None, query_args=None, + override_num_retries=None): + if mode == 'r': + self.mode = 'r' + self.open_read(headers=headers, query_args=query_args, + override_num_retries=override_num_retries) + elif mode == 'w': + self.mode = 'w' + self.open_write(headers=headers, + override_num_retries=override_num_retries) + else: + raise BotoClientError('Invalid mode: %s' % mode) + + closed = False + def close(self): + if self.resp: + self.resp.read() + self.resp = None + self.mode = None + self.closed = True + + def next(self): + """ + By providing a next method, the key object supports use as an iterator. + For example, you can now say: + + for bytes in key: + write bytes to a file or whatever + + All of the HTTP connection stuff is handled for you. + """ + self.open_read() + data = self.resp.read(self.BufferSize) + if not data: + self.close() + raise StopIteration + return data + + def read(self, size=0): + if size == 0: + size = self.BufferSize + self.open_read() + data = self.resp.read(size) + if not data: + self.close() + return data + + def change_storage_class(self, new_storage_class, dst_bucket=None): + """ + Change the storage class of an existing key. + Depending on whether a different destination bucket is supplied + or not, this will either move the item within the bucket, preserving + all metadata and ACL info bucket changing the storage class or it + will copy the item to the provided destination bucket, also + preserving metadata and ACL info. + + :type new_storage_class: string + :param new_storage_class: The new storage class for the Key. + Possible values are: + * STANDARD + * REDUCED_REDUNDANCY + + :type dst_bucket: string + :param dst_bucket: The name of a destination bucket. If not + provided the current bucket of the key + will be used. + + """ + if new_storage_class == 'STANDARD': + return self.copy(self.bucket.name, self.name, + reduced_redundancy=False, preserve_acl=True) + elif new_storage_class == 'REDUCED_REDUNDANCY': + return self.copy(self.bucket.name, self.name, + reduced_redundancy=True, preserve_acl=True) + else: + raise BotoClientError('Invalid storage class: %s' % + new_storage_class) + + def copy(self, dst_bucket, dst_key, metadata=None, + reduced_redundancy=False, preserve_acl=False): + """ + Copy this Key to another bucket. + + :type dst_bucket: string + :param dst_bucket: The name of the destination bucket + + :type dst_key: string + :param dst_key: The name of the destination key + + :type metadata: dict + :param metadata: Metadata to be associated with new key. + If metadata is supplied, it will replace the + metadata of the source key being copied. + If no metadata is supplied, the source key's + metadata will be copied to the new key. + + :type reduced_redundancy: bool + :param reduced_redundancy: If True, this will force the storage + class of the new Key to be + REDUCED_REDUNDANCY regardless of the + storage class of the key being copied. + The Reduced Redundancy Storage (RRS) + feature of S3, provides lower + redundancy at lower storage cost. + + :type preserve_acl: bool + :param preserve_acl: If True, the ACL from the source key + will be copied to the destination + key. If False, the destination key + will have the default ACL. + Note that preserving the ACL in the + new key object will require two + additional API calls to S3, one to + retrieve the current ACL and one to + set that ACL on the new object. If + you don't care about the ACL, a value + of False will be significantly more + efficient. + + :rtype: :class:`boto.s3.key.Key` or subclass + :returns: An instance of the newly created key object + """ + dst_bucket = self.bucket.connection.lookup(dst_bucket) + if reduced_redundancy: + storage_class = 'REDUCED_REDUNDANCY' + else: + storage_class = self.storage_class + return dst_bucket.copy_key(dst_key, self.bucket.name, + self.name, metadata, + storage_class=storage_class, + preserve_acl=preserve_acl) + + def startElement(self, name, attrs, connection): + if name == 'Owner': + self.owner = User(self) + return self.owner + else: + return None + + def endElement(self, name, value, connection): + if name == 'Key': + self.name = value.encode('utf-8') + elif name == 'ETag': + self.etag = value + elif name == 'LastModified': + self.last_modified = value + elif name == 'Size': + self.size = int(value) + elif name == 'StorageClass': + self.storage_class = value + elif name == 'Owner': + pass + elif name == 'VersionId': + self.version_id = value + else: + setattr(self, name, value) + + def exists(self): + """ + Returns True if the key exists + + :rtype: bool + :return: Whether the key exists on S3 + """ + return bool(self.bucket.lookup(self.name)) + + def delete(self): + """ + Delete this key from S3 + """ + return self.bucket.delete_key(self.name, version_id=self.version_id) + + def get_metadata(self, name): + return self.metadata.get(name) + + def set_metadata(self, name, value): + self.metadata[name] = value + + def update_metadata(self, d): + self.metadata.update(d) + + # convenience methods for setting/getting ACL + def set_acl(self, acl_str, headers=None): + if self.bucket != None: + self.bucket.set_acl(acl_str, self.name, headers=headers) + + def get_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_acl(self.name, headers=headers) + + def get_xml_acl(self, headers=None): + if self.bucket != None: + return self.bucket.get_xml_acl(self.name, headers=headers) + + def set_xml_acl(self, acl_str, headers=None): + if self.bucket != None: + return self.bucket.set_xml_acl(acl_str, self.name, headers=headers) + + def set_canned_acl(self, acl_str, headers=None): + return self.bucket.set_canned_acl(acl_str, self.name, headers) + + def make_public(self, headers=None): + return self.bucket.set_canned_acl('public-read', self.name, headers) + + def generate_url(self, expires_in, method='GET', headers=None, + query_auth=True, force_http=False): + """ + Generate a URL to access this key. + + :type expires_in: int + :param expires_in: How long the url is valid for, in seconds + + :type method: string + :param method: The method to use for retrieving the file (default is GET) + + :type headers: dict + :param headers: Any headers to pass along in the request + + :type query_auth: bool + :param query_auth: + + :rtype: string + :return: The URL to access the key + """ + return self.bucket.connection.generate_url(expires_in, method, + self.bucket.name, self.name, + headers, query_auth, force_http) + + def send_file(self, fp, headers=None, cb=None, num_cb=10, query_args=None): + """ + Upload a file to a key into a bucket on S3. + + :type fp: file + :param fp: The file pointer to upload + + :type headers: dict + :param headers: The headers to pass along with the PUT request + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the cb + parameter this parameter determines the granularity + of the callback by defining the maximum number of + times the callback will be called during the file + transfer. Providing a negative integer will cause + your callback to be called with each buffer read. + + """ + provider = self.bucket.connection.provider + + def sender(http_conn, method, path, data, headers): + http_conn.putrequest(method, path) + for key in headers: + http_conn.putheader(key, headers[key]) + http_conn.endheaders() + fp.seek(0) + save_debug = self.bucket.connection.debug + self.bucket.connection.debug = 0 + http_conn.set_debuglevel(0) + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + elif num_cb < 0: + cb_count = -1 + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + l = fp.read(self.BufferSize) + while len(l) > 0: + http_conn.send(l) + if cb: + total_bytes += len(l) + i += 1 + if i == cb_count or cb_count == -1: + cb(total_bytes, self.size) + i = 0 + l = fp.read(self.BufferSize) + if cb: + cb(total_bytes, self.size) + response = http_conn.getresponse() + body = response.read() + fp.seek(0) + http_conn.set_debuglevel(save_debug) + self.bucket.connection.debug = save_debug + if response.status == 500 or response.status == 503 or \ + response.getheader('location'): + # we'll try again + return response + elif response.status >= 200 and response.status <= 299: + self.etag = response.getheader('etag') + if self.etag != '"%s"' % self.md5: + raise provider.storage_data_error( + 'ETag from S3 did not match computed MD5') + return response + else: + raise provider.storage_response_error( + response.status, response.reason, body) + + if not headers: + headers = {} + else: + headers = headers.copy() + headers['User-Agent'] = UserAgent + headers['Content-MD5'] = self.base64md5 + if self.storage_class != 'STANDARD': + headers[provider.storage_class_header] = self.storage_class + if headers.has_key('Content-Encoding'): + self.content_encoding = headers['Content-Encoding'] + if headers.has_key('Content-Type'): + self.content_type = headers['Content-Type'] + elif self.path: + self.content_type = mimetypes.guess_type(self.path)[0] + if self.content_type == None: + self.content_type = self.DefaultContentType + headers['Content-Type'] = self.content_type + else: + headers['Content-Type'] = self.content_type + headers['Content-Length'] = str(self.size) + headers['Expect'] = '100-Continue' + headers = boto.utils.merge_meta(headers, self.metadata, provider) + resp = self.bucket.connection.make_request('PUT', self.bucket.name, + self.name, headers, + sender=sender, + query_args=query_args) + self.handle_version_headers(resp, force=True) + + def compute_md5(self, fp): + """ + :type fp: file + :param fp: File pointer to the file to MD5 hash. The file pointer will be + reset to the beginning of the file before the method returns. + + :rtype: tuple + :return: A tuple containing the hex digest version of the MD5 hash + as the first element and the base64 encoded version of the + plain digest as the second element. + """ + m = md5() + fp.seek(0) + s = fp.read(self.BufferSize) + while s: + m.update(s) + s = fp.read(self.BufferSize) + hex_md5 = m.hexdigest() + base64md5 = base64.encodestring(m.digest()) + if base64md5[-1] == '\n': + base64md5 = base64md5[0:-1] + self.size = fp.tell() + fp.seek(0) + return (hex_md5, base64md5) + + def set_contents_from_file(self, fp, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + reduced_redundancy=False, query_args=None): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file pointed to by 'fp' as the + contents. + + :type fp: file + :param fp: the file whose contents to upload + + :type headers: dict + :param headers: additional HTTP headers that will be sent with the PUT request. + + :type replace: bool + :param replace: If this parameter is False, the method + will first check to see if an object exists in the + bucket with the same key. If it does, it won't + overwrite it. The default value is True which will + overwrite the object. + + :type cb: function + :param cb: a callback function that will be called to report + progress on the upload. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted to S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + :type reduced_redundancy: bool + :param reduced_redundancy: If True, this will set the storage + class of the new Key to be + REDUCED_REDUNDANCY. The Reduced Redundancy + Storage (RRS) feature of S3, provides lower + redundancy at lower storage cost. + + """ + provider = self.bucket.connection.provider + if headers is None: + headers = {} + if policy: + headers[provider.acl_header] = policy + if reduced_redundancy: + self.storage_class = 'REDUCED_REDUNDANCY' + if provider.storage_class_header: + headers[provider.storage_class_header] = self.storage_class + # TODO - What if the provider doesn't support reduced reduncancy? + # What if different providers provide different classes? + if hasattr(fp, 'name'): + self.path = fp.name + if self.bucket != None: + if not md5: + md5 = self.compute_md5(fp) + else: + # even if md5 is provided, still need to set size of content + fp.seek(0, 2) + self.size = fp.tell() + fp.seek(0) + self.md5 = md5[0] + self.base64md5 = md5[1] + if self.name == None: + self.name = self.md5 + if not replace: + k = self.bucket.lookup(self.name) + if k: + return + self.send_file(fp, headers, cb, num_cb, query_args) + + def set_contents_from_filename(self, filename, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + reduced_redundancy=False): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the contents of the file named by 'filename'. + See set_contents_from_file method for details about the + parameters. + + :type filename: string + :param filename: The name of the file that you want to put onto S3 + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + + :type reduced_redundancy: bool + :param reduced_redundancy: If True, this will set the storage + class of the new Key to be + REDUCED_REDUNDANCY. The Reduced Redundancy + Storage (RRS) feature of S3, provides lower + redundancy at lower storage cost. + """ + fp = open(filename, 'rb') + self.set_contents_from_file(fp, headers, replace, cb, num_cb, + policy, md5, reduced_redundancy) + fp.close() + + def set_contents_from_string(self, s, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + reduced_redundancy=False): + """ + Store an object in S3 using the name of the Key object as the + key in S3 and the string 's' as the contents. + See set_contents_from_file method for details about the + parameters. + + :type headers: dict + :param headers: Additional headers to pass along with the request to AWS. + + :type replace: bool + :param replace: If True, replaces the contents of the file if it already exists. + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type policy: :class:`boto.s3.acl.CannedACLStrings` + :param policy: A canned ACL policy that will be applied to the new key in S3. + + :type md5: A tuple containing the hexdigest version of the MD5 checksum of the + file as the first element and the Base64-encoded version of the plain + checksum as the second element. This is the same format returned by + the compute_md5 method. + :param md5: If you need to compute the MD5 for any reason prior to upload, + it's silly to have to do it twice so this param, if present, will be + used as the MD5 values of the file. Otherwise, the checksum will be computed. + + :type reduced_redundancy: bool + :param reduced_redundancy: If True, this will set the storage + class of the new Key to be + REDUCED_REDUNDANCY. The Reduced Redundancy + Storage (RRS) feature of S3, provides lower + redundancy at lower storage cost. + """ + fp = StringIO.StringIO(s) + r = self.set_contents_from_file(fp, headers, replace, cb, num_cb, + policy, md5, reduced_redundancy) + fp.close() + return r + + def get_file(self, fp, headers=None, cb=None, num_cb=10, + torrent=False, version_id=None, override_num_retries=None, + response_headers=None): + """ + Retrieves a file from an S3 Key + + :type fp: file + :param fp: File pointer to put the data into + + :type headers: string + :param: headers to send when retrieving the files + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from S3 and the second representing + the total number of bytes that need to be transmitted. + + + :type cb: int + :param num_cb: (optional) If a callback is specified with the cb parameter + this parameter determines the granularity of the callback by defining + the maximum number of times the callback will be called during the file transfer. + + :type torrent: bool + :param torrent: Flag for whether to get a torrent for the file + + :type override_num_retries: int + :param override_num_retries: If not None will override configured + num_retries parameter for underlying GET. + + :type response_headers: dict + :param response_headers: A dictionary containing HTTP headers/values + that will override any headers associated with + the stored object in the response. + See http://goo.gl/EWOPb for details. + """ + if cb: + if num_cb > 2: + cb_count = self.size / self.BufferSize / (num_cb-2) + elif num_cb < 0: + cb_count = -1 + else: + cb_count = 0 + i = total_bytes = 0 + cb(total_bytes, self.size) + save_debug = self.bucket.connection.debug + if self.bucket.connection.debug == 1: + self.bucket.connection.debug = 0 + + query_args = [] + if torrent: + query_args.append('torrent') + # If a version_id is passed in, use that. If not, check to see + # if the Key object has an explicit version_id and, if so, use that. + # Otherwise, don't pass a version_id query param. + if version_id is None: + version_id = self.version_id + if version_id: + query_args.append('versionId=%s' % version_id) + if response_headers: + for key in response_headers: + query_args.append('%s=%s' % (key, response_headers[key])) + query_args = '&'.join(query_args) + self.open('r', headers, query_args=query_args, + override_num_retries=override_num_retries) + for bytes in self: + fp.write(bytes) + if cb: + total_bytes += len(bytes) + i += 1 + if i == cb_count or cb_count == -1: + cb(total_bytes, self.size) + i = 0 + if cb: + cb(total_bytes, self.size) + self.close() + self.bucket.connection.debug = save_debug + + def get_torrent_file(self, fp, headers=None, cb=None, num_cb=10): + """ + Get a torrent file (see to get_file) + + :type fp: file + :param fp: The file pointer of where to put the torrent + + :type headers: dict + :param headers: Headers to be passed + + :type cb: function + :param cb: (optional) a callback function that will be called to + report progress on the download. The callback should + accept two integer parameters, the first representing + the number of bytes that have been successfully + transmitted from S3 and the second representing the + total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the + cb parameter this parameter determines the + granularity of the callback by defining the + maximum number of times the callback will be + called during the file transfer. + + """ + return self.get_file(fp, headers, cb, num_cb, torrent=True) + + def get_contents_to_file(self, fp, headers=None, + cb=None, num_cb=10, + torrent=False, + version_id=None, + res_download_handler=None, + response_headers=None): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Write the contents of the object to the file pointed + to by 'fp'. + + :type fp: File -like object + :param fp: + + :type headers: dict + :param headers: additional HTTP headers that will be sent with + the GET request. + + :type cb: function + :param cb: (optional) a callback function that will be called to + report progress on the download. The callback should + accept two integer parameters, the first representing + the number of bytes that have been successfully + transmitted from S3 and the second representing the + total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the + cb parameter this parameter determines the + granularity of the callback by defining the + maximum number of times the callback will be + called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent + file as a string. + + :type res_upload_handler: ResumableDownloadHandler + :param res_download_handler: If provided, this handler will + perform the download. + + :type response_headers: dict + :param response_headers: A dictionary containing HTTP headers/values + that will override any headers associated with + the stored object in the response. + See http://goo.gl/EWOPb for details. + """ + if self.bucket != None: + if res_download_handler: + res_download_handler.get_file(self, fp, headers, cb, num_cb, + torrent=torrent, + version_id=version_id) + else: + self.get_file(fp, headers, cb, num_cb, torrent=torrent, + version_id=version_id, + response_headers=response_headers) + + def get_contents_to_filename(self, filename, headers=None, + cb=None, num_cb=10, + torrent=False, + version_id=None, + res_download_handler=None, + response_headers=None): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Store contents of the object to a file named by 'filename'. + See get_contents_to_file method for details about the + parameters. + + :type filename: string + :param filename: The filename of where to put the file contents + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to + report progress on the download. The callback should + accept two integer parameters, the first representing + the number of bytes that have been successfully + transmitted from S3 and the second representing the + total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the + cb parameter this parameter determines the + granularity of the callback by defining the + maximum number of times the callback will be + called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file + as a string. + + :type res_upload_handler: ResumableDownloadHandler + :param res_download_handler: If provided, this handler will + perform the download. + + :type response_headers: dict + :param response_headers: A dictionary containing HTTP headers/values + that will override any headers associated with + the stored object in the response. + See http://goo.gl/EWOPb for details. + """ + fp = open(filename, 'wb') + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, + version_id=version_id, + res_download_handler=res_download_handler, + response_headers=response_headers) + fp.close() + # if last_modified date was sent from s3, try to set file's timestamp + if self.last_modified != None: + try: + modified_tuple = rfc822.parsedate_tz(self.last_modified) + modified_stamp = int(rfc822.mktime_tz(modified_tuple)) + os.utime(fp.name, (modified_stamp, modified_stamp)) + except Exception: pass + + def get_contents_as_string(self, headers=None, + cb=None, num_cb=10, + torrent=False, + version_id=None, + response_headers=None): + """ + Retrieve an object from S3 using the name of the Key object as the + key in S3. Return the contents of the object as a string. + See get_contents_to_file method for details about the + parameters. + + :type headers: dict + :param headers: Any additional headers to send in the request + + :type cb: function + :param cb: (optional) a callback function that will be called to + report progress on the download. The callback should + accept two integer parameters, the first representing + the number of bytes that have been successfully + transmitted from S3 and the second representing the + total number of bytes that need to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the + cb parameter this parameter determines the + granularity of the callback by defining the + maximum number of times the callback will be + called during the file transfer. + + :type torrent: bool + :param torrent: If True, returns the contents of a torrent file + as a string. + + :type response_headers: dict + :param response_headers: A dictionary containing HTTP headers/values + that will override any headers associated with + the stored object in the response. + See http://goo.gl/EWOPb for details. + + :rtype: string + :returns: The contents of the file as a string + """ + fp = StringIO.StringIO() + self.get_contents_to_file(fp, headers, cb, num_cb, torrent=torrent, + version_id=version_id, + response_headers=response_headers) + return fp.getvalue() + + def add_email_grant(self, permission, email_address, headers=None): + """ + Convenience method that provides a quick way to add an email grant + to a key. This method retrieves the current ACL, creates a new + grant based on the parameters passed in, adds that grant to the ACL + and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + + :type email_address: string + :param email_address: The email address associated with the AWS + account your are granting the permission to. + + :type recursive: boolean + :param recursive: A boolean value to controls whether the command + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + policy = self.get_acl(headers=headers) + policy.acl.add_email_grant(permission, email_address) + self.set_acl(policy, headers=headers) + + def add_user_grant(self, permission, user_id, headers=None): + """ + Convenience method that provides a quick way to add a canonical + user grant to a key. This method retrieves the current ACL, + creates a new grant based on the parameters passed in, adds that + grant to the ACL and then PUT's the new ACL back to S3. + + :type permission: string + :param permission: The permission being granted. Should be one of: + (READ, WRITE, READ_ACP, WRITE_ACP, FULL_CONTROL). + + :type user_id: string + :param user_id: The canonical user id associated with the AWS + account your are granting the permission to. + + :type recursive: boolean + :param recursive: A boolean value to controls whether the command + will apply the grant to all keys within the bucket + or not. The default value is False. By passing a + True value, the call will iterate through all keys + in the bucket and apply the same grant to each key. + CAUTION: If you have a lot of keys, this could take + a long time! + """ + policy = self.get_acl() + policy.acl.add_user_grant(permission, user_id) + self.set_acl(policy, headers=headers) diff --git a/backup/src/boto/s3/multipart.py b/backup/src/boto/s3/multipart.py new file mode 100644 index 0000000..c6160de --- /dev/null +++ b/backup/src/boto/s3/multipart.py @@ -0,0 +1,259 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import user +import key +from boto import handler +import xml.sax + +class CompleteMultiPartUpload(object): + """ + Represents a completed MultiPart Upload. Contains the + following useful attributes: + + * location - The URI of the completed upload + * bucket_name - The name of the bucket in which the upload + is contained + * key_name - The name of the new, completed key + * etag - The MD5 hash of the completed, combined upload + """ + + def __init__(self, bucket=None): + self.bucket = None + self.location = None + self.bucket_name = None + self.key_name = None + self.etag = None + + def __repr__(self): + return '' % (self.bucket_name, + self.key_name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Location': + self.location = value + elif name == 'Bucket': + self.bucket_name = value + elif name == 'Key': + self.key_name = value + elif name == 'ETag': + self.etag = value + else: + setattr(self, name, value) + +class Part(object): + """ + Represents a single part in a MultiPart upload. + Attributes include: + + * part_number - The integer part number + * last_modified - The last modified date of this part + * etag - The MD5 hash of this part + * size - The size, in bytes, of this part + """ + + def __init__(self, bucket=None): + self.bucket = bucket + self.part_number = None + self.last_modified = None + self.etag = None + self.size = None + + def __repr__(self): + if isinstance(self.part_number, int): + return '' % self.part_number + else: + return '' % None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'PartNumber': + self.part_number = int(value) + elif name == 'LastModified': + self.last_modified = value + elif name == 'ETag': + self.etag = value + elif name == 'Size': + self.size = int(value) + else: + setattr(self, name, value) + +def part_lister(mpupload, part_number_marker=''): + """ + A generator function for listing parts of a multipart upload. + """ + more_results = True + part = None + while more_results: + parts = mpupload.get_all_parts(None, part_number_marker) + for part in parts: + yield part + part_number_marker = mpupload.next_part_number_marker + more_results= mpupload.is_truncated + +class MultiPartUpload(object): + """ + Represents a MultiPart Upload operation. + """ + + def __init__(self, bucket=None): + self.bucket = bucket + self.bucket_name = None + self.key_name = None + self.id = id + self.initiator = None + self.owner = None + self.storage_class = None + self.initiated = None + self.part_number_marker = None + self.next_part_number_marker = None + self.max_parts = None + self.is_truncated = False + self._parts = None + + def __repr__(self): + return '' % self.key_name + + def __iter__(self): + return part_lister(self, part_number_marker=self.part_number_marker) + + def to_xml(self): + self.get_all_parts() + s = '\n' + for part in self: + s += ' \n' + s += ' %d\n' % part.part_number + s += ' %s\n' % part.etag + s += ' \n' + s += '' + return s + + def startElement(self, name, attrs, connection): + if name == 'Initiator': + self.initiator = user.User(self) + return self.initiator + elif name == 'Owner': + self.owner = user.User(self) + return self.owner + elif name == 'Part': + part = Part(self.bucket) + self._parts.append(part) + return part + return None + + def endElement(self, name, value, connection): + if name == 'Bucket': + self.bucket_name = value + elif name == 'Key': + self.key_name = value + elif name == 'UploadId': + self.id = value + elif name == 'StorageClass': + self.storage_class = value + elif name == 'PartNumberMarker': + self.part_number_marker = value + elif name == 'NextPartNumberMarker': + self.next_part_number_marker = value + elif name == 'MaxParts': + self.max_parts = int(value) + elif name == 'IsTruncated': + if value == 'true': + self.is_truncated = True + else: + self.is_truncated = False + else: + setattr(self, name, value) + + def get_all_parts(self, max_parts=None, part_number_marker=None): + """ + Return the uploaded parts of this MultiPart Upload. This is + a lower-level method that requires you to manually page through + results. To simplify this process, you can just use the + object itself as an iterator and it will automatically handle + all of the paging with S3. + """ + self._parts = [] + query_args = 'uploadId=%s' % self.id + if max_parts: + query_args += '&max_parts=%d' % max_parts + if part_number_marker: + query_args += '&part-number-marker=%s' % part_number_marker + response = self.bucket.connection.make_request('GET', self.bucket.name, + self.key_name, + query_args=query_args) + body = response.read() + if response.status == 200: + h = handler.XmlHandler(self, self) + xml.sax.parseString(body, h) + return self._parts + + def upload_part_from_file(self, fp, part_num, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None): + """ + Upload another part of this MultiPart Upload. + + :type fp: file + :param fp: The file object you want to upload. + + :type part_num: int + :param part_num: The number of this part. + + The other parameters are exactly as defined for the + :class:`boto.s3.key.Key` set_contents_from_file method. + """ + if part_num < 1: + raise ValueError('Part numbers must be greater than zero') + query_args = 'uploadId=%s&partNumber=%d' % (self.id, part_num) + key = self.bucket.new_key(self.key_name) + key.set_contents_from_file(fp, headers, replace, cb, num_cb, policy, + md5, reduced_redundancy=False, query_args=query_args) + + def complete_upload(self): + """ + Complete the MultiPart Upload operation. This method should + be called when all parts of the file have been successfully + uploaded to S3. + + :rtype: :class:`boto.s3.multipart.CompletedMultiPartUpload` + :returns: An object representing the completed upload. + """ + xml = self.to_xml() + self.bucket.complete_multipart_upload(self.key_name, + self.id, xml) + + def cancel_upload(self): + """ + Cancels a MultiPart Upload operation. The storage consumed by + any previously uploaded parts will be freed. However, if any + part uploads are currently in progress, those part uploads + might or might not succeed. As a result, it might be necessary + to abort a given multipart upload multiple times in order to + completely free all storage consumed by all parts. + """ + self.bucket.cancel_multipart_upload(self.key_name, self.id) + + diff --git a/backup/src/boto/s3/prefix.py b/backup/src/boto/s3/prefix.py new file mode 100644 index 0000000..fc0f26a --- /dev/null +++ b/backup/src/boto/s3/prefix.py @@ -0,0 +1,35 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Prefix: + def __init__(self, bucket=None, name=None): + self.bucket = bucket + self.name = name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'Prefix': + self.name = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/s3/resumable_download_handler.py b/backup/src/boto/s3/resumable_download_handler.py new file mode 100644 index 0000000..0d01477 --- /dev/null +++ b/backup/src/boto/s3/resumable_download_handler.py @@ -0,0 +1,330 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import errno +import httplib +import os +import re +import socket +import time +import boto +from boto import config, storage_uri_for_key +from boto.connection import AWSAuthConnection +from boto.exception import ResumableDownloadException +from boto.exception import ResumableTransferDisposition + +""" +Resumable download handler. + +Resumable downloads will retry failed downloads, resuming at the byte count +completed by the last download attempt. If too many retries happen with no +progress (per configurable num_retries param), the download will be aborted. + +The caller can optionally specify a tracker_file_name param in the +ResumableDownloadHandler constructor. If you do this, that file will +save the state needed to allow retrying later, in a separate process +(e.g., in a later run of gsutil). + +Note that resumable downloads work across providers (they depend only +on support Range GETs), but this code is in the boto.s3 package +because it is the wrong abstraction level to go in the top-level boto +package. + +TODO: At some point we should refactor the code to have a storage_service +package where all these provider-independent files go. +""" + + +class ByteTranslatingCallbackHandler(object): + """ + Proxy class that translates progress callbacks made by + boto.s3.Key.get_file(), taking into account that we're resuming + a download. + """ + def __init__(self, proxied_cb, download_start_point): + self.proxied_cb = proxied_cb + self.download_start_point = download_start_point + + def call(self, total_bytes_uploaded, total_size): + self.proxied_cb(self.download_start_point + total_bytes_uploaded, + self.download_start_point + total_size) + + +def get_cur_file_size(fp, position_to_eof=False): + """ + Returns size of file, optionally leaving fp positioned at EOF. + """ + if not position_to_eof: + cur_pos = fp.tell() + fp.seek(0, os.SEEK_END) + cur_file_size = fp.tell() + if not position_to_eof: + fp.seek(cur_pos, os.SEEK_SET) + return cur_file_size + + +class ResumableDownloadHandler(object): + """ + Handler for resumable downloads. + """ + + ETAG_REGEX = '([a-z0-9]{32})\n' + + RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error, + socket.gaierror) + + def __init__(self, tracker_file_name=None, num_retries=None): + """ + Constructor. Instantiate once for each downloaded file. + + :type tracker_file_name: string + :param tracker_file_name: optional file name to save tracking info + about this download. If supplied and the current process fails + the download, it can be retried in a new process. If called + with an existing file containing an unexpired timestamp, + we'll resume the transfer for this file; else we'll start a + new resumable download. + + :type num_retries: int + :param num_retries: the number of times we'll re-try a resumable + download making no progress. (Count resets every time we get + progress, so download can span many more than this number of + retries.) + """ + self.tracker_file_name = tracker_file_name + self.num_retries = num_retries + self.etag_value_for_current_download = None + if tracker_file_name: + self._load_tracker_file_etag() + # Save download_start_point in instance state so caller can + # find how much was transferred by this ResumableDownloadHandler + # (across retries). + self.download_start_point = None + + def _load_tracker_file_etag(self): + f = None + try: + f = open(self.tracker_file_name, 'r') + etag_line = f.readline() + m = re.search(self.ETAG_REGEX, etag_line) + if m: + self.etag_value_for_current_download = m.group(1) + else: + print('Couldn\'t read etag in tracker file (%s). Restarting ' + 'download from scratch.' % self.tracker_file_name) + except IOError, e: + # Ignore non-existent file (happens first time a download + # is attempted on an object), but warn user for other errors. + if e.errno != errno.ENOENT: + # Will restart because + # self.etag_value_for_current_download == None. + print('Couldn\'t read URI tracker file (%s): %s. Restarting ' + 'download from scratch.' % + (self.tracker_file_name, e.strerror)) + finally: + if f: + f.close() + + def _save_tracker_info(self, key): + self.etag_value_for_current_download = key.etag.strip('"\'') + if not self.tracker_file_name: + return + f = None + try: + f = open(self.tracker_file_name, 'w') + f.write('%s\n' % self.etag_value_for_current_download) + except IOError, e: + raise ResumableDownloadException( + 'Couldn\'t write tracker file (%s): %s.\nThis can happen' + 'if you\'re using an incorrectly configured download tool\n' + '(e.g., gsutil configured to save tracker files to an ' + 'unwritable directory)' % + (self.tracker_file_name, e.strerror), + ResumableTransferDisposition.ABORT) + finally: + if f: + f.close() + + def _remove_tracker_file(self): + if (self.tracker_file_name and + os.path.exists(self.tracker_file_name)): + os.unlink(self.tracker_file_name) + + def _attempt_resumable_download(self, key, fp, headers, cb, num_cb, + torrent, version_id): + """ + Attempts a resumable download. + + Raises ResumableDownloadException if any problems occur. + """ + cur_file_size = get_cur_file_size(fp, position_to_eof=True) + + if (cur_file_size and + self.etag_value_for_current_download and + self.etag_value_for_current_download == key.etag.strip('"\'')): + # Try to resume existing transfer. + if cur_file_size > key.size: + raise ResumableDownloadException( + '%s is larger (%d) than %s (%d).\nDeleting tracker file, so ' + 'if you re-try this download it will start from scratch' % + (fp.name, cur_file_size, str(storage_uri_for_key(key)), + key.size), ResumableTransferDisposition.ABORT) + elif cur_file_size == key.size: + if key.bucket.connection.debug >= 1: + print 'Download complete.' + return + if key.bucket.connection.debug >= 1: + print 'Resuming download.' + headers = headers.copy() + headers['Range'] = 'bytes=%d-%d' % (cur_file_size, key.size - 1) + cb = ByteTranslatingCallbackHandler(cb, cur_file_size).call + self.download_start_point = cur_file_size + else: + if key.bucket.connection.debug >= 1: + print 'Starting new resumable download.' + self._save_tracker_info(key) + self.download_start_point = 0 + # Truncate the file, in case a new resumable download is being + # started atop an existing file. + fp.truncate(0) + + # Disable AWSAuthConnection-level retry behavior, since that would + # cause downloads to restart from scratch. + key.get_file(fp, headers, cb, num_cb, torrent, version_id, + override_num_retries=0) + fp.flush() + + def _check_final_md5(self, key, file_name): + """ + Checks that etag from server agrees with md5 computed after the + download completes. This is important, since the download could + have spanned a number of hours and multiple processes (e.g., + gsutil runs), and the user could change some of the file and not + realize they have inconsistent data. + """ + fp = open(file_name, 'r') + if key.bucket.connection.debug >= 1: + print 'Checking md5 against etag.' + hex_md5 = key.compute_md5(fp)[0] + if hex_md5 != key.etag.strip('"\''): + file_name = fp.name + fp.close() + os.unlink(file_name) + raise ResumableDownloadException( + 'File changed during download: md5 signature doesn\'t match ' + 'etag (incorrect downloaded file deleted)', + ResumableTransferDisposition.ABORT) + + def get_file(self, key, fp, headers, cb=None, num_cb=10, torrent=False, + version_id=None): + """ + Retrieves a file from a Key + :type key: :class:`boto.s3.key.Key` or subclass + :param key: The Key object from which upload is to be downloaded + + :type fp: file + :param fp: File pointer into which data should be downloaded + + :type headers: string + :param: headers to send when retrieving the files + + :type cb: function + :param cb: (optional) a callback function that will be called to report + progress on the download. The callback should accept two integer + parameters, the first representing the number of bytes that have + been successfully transmitted from the storage service and + the second representing the total number of bytes that need + to be transmitted. + + :type num_cb: int + :param num_cb: (optional) If a callback is specified with the cb + parameter this parameter determines the granularity of the callback + by defining the maximum number of times the callback will be + called during the file transfer. + + :type torrent: bool + :param torrent: Flag for whether to get a torrent for the file + + :type version_id: string + :param version_id: The version ID (optional) + + Raises ResumableDownloadException if a problem occurs during + the transfer. + """ + + debug = key.bucket.connection.debug + if not headers: + headers = {} + + # Use num-retries from constructor if one was provided; else check + # for a value specified in the boto config file; else default to 5. + if self.num_retries is None: + self.num_retries = config.getint('Boto', 'num_retries', 5) + progress_less_iterations = 0 + + while True: # Retry as long as we're making progress. + had_file_bytes_before_attempt = get_cur_file_size(fp) + try: + self._attempt_resumable_download(key, fp, headers, cb, num_cb, + torrent, version_id) + # Download succceded, so remove the tracker file (if have one). + self._remove_tracker_file() + self._check_final_md5(key, fp.name) + if debug >= 1: + print 'Resumable download complete.' + return + except self.RETRYABLE_EXCEPTIONS, e: + if debug >= 1: + print('Caught exception (%s)' % e.__repr__()) + except ResumableDownloadException, e: + if e.disposition == ResumableTransferDisposition.ABORT: + if debug >= 1: + print('Caught non-retryable ResumableDownloadException ' + '(%s)' % e.message) + raise + else: + if debug >= 1: + print('Caught ResumableDownloadException (%s) - will ' + 'retry' % e.message) + + # At this point we had a re-tryable failure; see if made progress. + if get_cur_file_size(fp) > had_file_bytes_before_attempt: + progress_less_iterations = 0 + else: + progress_less_iterations += 1 + + if progress_less_iterations > self.num_retries: + # Don't retry any longer in the current process. + raise ResumableDownloadException( + 'Too many resumable download attempts failed without ' + 'progress. You might try this download again later', + ResumableTransferDisposition.ABORT) + + # Close the key, in case a previous download died partway + # through and left data in the underlying key HTTP buffer. + key.close() + + sleep_time_secs = 2**progress_less_iterations + if debug >= 1: + print('Got retryable failure (%d progress-less in a row).\n' + 'Sleeping %d seconds before re-trying' % + (progress_less_iterations, sleep_time_secs)) + time.sleep(sleep_time_secs) diff --git a/backup/src/boto/s3/user.py b/backup/src/boto/s3/user.py new file mode 100644 index 0000000..f45f038 --- /dev/null +++ b/backup/src/boto/s3/user.py @@ -0,0 +1,49 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class User: + def __init__(self, parent=None, id='', display_name=''): + if parent: + parent.owner = self + self.type = None + self.id = id + self.display_name = display_name + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DisplayName': + self.display_name = value + elif name == 'ID': + self.id = value + else: + setattr(self, name, value) + + def to_xml(self, element_name='Owner'): + if self.type: + s = '<%s xsi:type="%s">' % (element_name, self.type) + else: + s = '<%s>' % element_name + s += '%s' % self.id + s += '%s' % self.display_name + s += '' % element_name + return s diff --git a/backup/src/boto/sdb/__init__.py b/backup/src/boto/sdb/__init__.py new file mode 100644 index 0000000..f5642c1 --- /dev/null +++ b/backup/src/boto/sdb/__init__.py @@ -0,0 +1,56 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from regioninfo import SDBRegionInfo + +def regions(): + """ + Get all available regions for the SDB service. + + :rtype: list + :return: A list of :class:`boto.sdb.regioninfo.RegionInfo` instances + """ + return [SDBRegionInfo(name='us-east-1', + endpoint='sdb.amazonaws.com'), + SDBRegionInfo(name='eu-west-1', + endpoint='sdb.eu-west-1.amazonaws.com'), + SDBRegionInfo(name='us-west-1', + endpoint='sdb.us-west-1.amazonaws.com'), + SDBRegionInfo(name='ap-southeast-1', + endpoint='sdb.ap-southeast-1.amazonaws.com') + ] + +def connect_to_region(region_name): + """ + Given a valid region name, return a + :class:`boto.sdb.connection.SDBConnection`. + + :param str region_name: The name of the region to connect to. + + :rtype: :class:`boto.sdb.connection.SDBConnection` or ``None`` + :return: A connection to the given region, or None if an invalid region + name is given + """ + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/backup/src/boto/sdb/connection.py b/backup/src/boto/sdb/connection.py new file mode 100644 index 0000000..b5a45b8 --- /dev/null +++ b/backup/src/boto/sdb/connection.py @@ -0,0 +1,607 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import xml.sax +import threading +from boto import handler +from boto.connection import AWSQueryConnection +from boto.sdb.domain import Domain, DomainMetaData +from boto.sdb.item import Item +from boto.sdb.regioninfo import SDBRegionInfo +from boto.exception import SDBResponseError + +class ItemThread(threading.Thread): + """ + A threaded :class:`Item ` retriever utility class. + Retrieved :class:`Item ` objects are stored in the + ``items`` instance variable after + :py:meth:`run() ` is called. + + .. tip:: + The item retrieval will not start until the + :func:`run() ` method is called. + """ + def __init__(self, name, domain_name, item_names): + """ + :param str name: A thread name. Used for identification. + :param str domain_name: The name of a SimpleDB + :class:`Domain ` + :type item_names: string or list of strings + :param item_names: The name(s) of the items to retrieve from the specified + :class:`Domain `. + :ivar list items: A list of items retrieved. Starts as empty list. + """ + threading.Thread.__init__(self, name=name) + #print 'starting %s with %d items' % (name, len(item_names)) + self.domain_name = domain_name + self.conn = SDBConnection() + self.item_names = item_names + self.items = [] + + def run(self): + """ + Start the threaded retrieval of items. Populates the + ``items`` list with :class:`Item ` objects. + """ + for item_name in self.item_names: + item = self.conn.get_attributes(self.domain_name, item_name) + self.items.append(item) + +#boto.set_stream_logger('sdb') + +class SDBConnection(AWSQueryConnection): + """ + This class serves as a gateway to your SimpleDB region (defaults to + us-east-1). Methods within allow access to SimpleDB + :class:`Domain ` objects and their associated + :class:`Item ` objects. + + .. tip:: + While you may instantiate this class directly, it may be easier to + go through :py:func:`boto.connect_sdb`. + """ + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'sdb.amazonaws.com' + APIVersion = '2009-04-15' + ResponseError = SDBResponseError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/', + converter=None): + """ + For any keywords that aren't documented, refer to the parent class, + :py:class:`boto.connection.AWSAuthConnection`. You can avoid having + to worry about these keyword arguments by instantiating these objects + via :py:func:`boto.connect_sdb`. + + :type region: :class:`boto.sdb.regioninfo.SDBRegionInfo` + :keyword region: Explicitly specify a region. Defaults to ``us-east-1`` + if not specified. + """ + if not region: + region = SDBRegionInfo(self, self.DefaultRegionName, + self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, + aws_secret_access_key, + is_secure, port, proxy, + proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, + https_connection_factory, path) + self.box_usage = 0.0 + self.converter = converter + self.item_cls = Item + + def _required_auth_capability(self): + return ['sdb'] + + def set_item_cls(self, cls): + """ + While the default item class is :py:class:`boto.sdb.item.Item`, this + default may be overridden. Use this method to change a connection's + item class. + + :param object cls: The new class to set as this connection's item + class. See the default item class for inspiration as to what your + replacement should/could look like. + """ + self.item_cls = cls + + def _build_name_value_list(self, params, attributes, replace=False, + label='Attribute'): + keys = attributes.keys() + keys.sort() + i = 1 + for key in keys: + value = attributes[key] + if isinstance(value, list): + for v in value: + params['%s.%d.Name' % (label, i)] = key + if self.converter: + v = self.converter.encode(v) + params['%s.%d.Value' % (label, i)] = v + if replace: + params['%s.%d.Replace' % (label, i)] = 'true' + i += 1 + else: + params['%s.%d.Name' % (label, i)] = key + if self.converter: + value = self.converter.encode(value) + params['%s.%d.Value' % (label, i)] = value + if replace: + params['%s.%d.Replace' % (label, i)] = 'true' + i += 1 + + def _build_expected_value(self, params, expected_value): + params['Expected.1.Name'] = expected_value[0] + if expected_value[1] is True: + params['Expected.1.Exists'] = 'true' + elif expected_value[1] is False: + params['Expected.1.Exists'] = 'false' + else: + params['Expected.1.Value'] = expected_value[1] + + def _build_batch_list(self, params, items, replace=False): + item_names = items.keys() + i = 0 + for item_name in item_names: + params['Item.%d.ItemName' % i] = item_name + j = 0 + item = items[item_name] + if item is not None: + attr_names = item.keys() + for attr_name in attr_names: + value = item[attr_name] + if isinstance(value, list): + for v in value: + if self.converter: + v = self.converter.encode(v) + params['Item.%d.Attribute.%d.Name' % (i, j)] = attr_name + params['Item.%d.Attribute.%d.Value' % (i, j)] = v + if replace: + params['Item.%d.Attribute.%d.Replace' % (i, j)] = 'true' + j += 1 + else: + params['Item.%d.Attribute.%d.Name' % (i, j)] = attr_name + if self.converter: + value = self.converter.encode(value) + params['Item.%d.Attribute.%d.Value' % (i, j)] = value + if replace: + params['Item.%d.Attribute.%d.Replace' % (i, j)] = 'true' + j += 1 + i += 1 + + def _build_name_list(self, params, attribute_names): + i = 1 + attribute_names.sort() + for name in attribute_names: + params['Attribute.%d.Name' % i] = name + i += 1 + + def get_usage(self): + """ + Returns the BoxUsage (in USD) accumulated on this specific SDBConnection + instance. + + .. tip:: This can be out of date, and should only be treated as a + rough estimate. Also note that this estimate only applies to the + requests made on this specific connection instance. It is by + no means an account-wide estimate. + + :rtype: float + :return: The accumulated BoxUsage of all requests made on the connection. + """ + return self.box_usage + + def print_usage(self): + """ + Print the BoxUsage and approximate costs of all requests made on + this specific SDBConnection instance. + + .. tip:: This can be out of date, and should only be treated as a + rough estimate. Also note that this estimate only applies to the + requests made on this specific connection instance. It is by + no means an account-wide estimate. + """ + print 'Total Usage: %f compute seconds' % self.box_usage + cost = self.box_usage * 0.14 + print 'Approximate Cost: $%f' % cost + + def get_domain(self, domain_name, validate=True): + """ + Retrieves a :py:class:`boto.sdb.domain.Domain` object whose name + matches ``domain_name``. + + :param str domain_name: The name of the domain to retrieve + :keyword bool validate: When ``True``, check to see if the domain + actually exists. If ``False``, blindly return a + :py:class:`Domain ` object with the + specified name set. + + :raises: + :py:class:`boto.exception.SDBResponseError` if ``validate`` is + ``True`` and no match could be found. + + :rtype: :py:class:`boto.sdb.domain.Domain` + :return: The requested domain + """ + domain = Domain(self, domain_name) + if validate: + self.select(domain, """select * from `%s` limit 1""" % domain_name) + return domain + + def lookup(self, domain_name, validate=True): + """ + Lookup an existing SimpleDB domain. This differs from + :py:meth:`get_domain` in that ``None`` is returned if ``validate`` is + ``True`` and no match was found (instead of raising an exception). + + :param str domain_name: The name of the domain to retrieve + + :param bool validate: If ``True``, a ``None`` value will be returned + if the specified domain can't be found. If ``False``, a + :py:class:`Domain ` object will be dumbly + returned, regardless of whether it actually exists. + + :rtype: :class:`boto.sdb.domain.Domain` object or ``None`` + :return: The Domain object or ``None`` if the domain does not exist. + """ + try: + domain = self.get_domain(domain_name, validate) + except: + domain = None + return domain + + def get_all_domains(self, max_domains=None, next_token=None): + """ + Returns a :py:class:`boto.resultset.ResultSet` containing + all :py:class:`boto.sdb.domain.Domain` objects associated with + this connection's Access Key ID. + + :keyword int max_domains: Limit the returned + :py:class:`ResultSet ` to the specified + number of members. + :keyword str next_token: A token string that was returned in an + earlier call to this method as the ``next_token`` attribute + on the returned :py:class:`ResultSet ` + object. This attribute is set if there are more than Domains than + the value specified in the ``max_domains`` keyword. Pass the + ``next_token`` value from you earlier query in this keyword to + get the next 'page' of domains. + """ + params = {} + if max_domains: + params['MaxNumberOfDomains'] = max_domains + if next_token: + params['NextToken'] = next_token + return self.get_list('ListDomains', params, [('DomainName', Domain)]) + + def create_domain(self, domain_name): + """ + Create a SimpleDB domain. + + :type domain_name: string + :param domain_name: The name of the new domain + + :rtype: :class:`boto.sdb.domain.Domain` object + :return: The newly created domain + """ + params = {'DomainName':domain_name} + d = self.get_object('CreateDomain', params, Domain) + d.name = domain_name + return d + + def get_domain_and_name(self, domain_or_name): + """ + Given a ``str`` or :class:`boto.sdb.domain.Domain`, return a + ``tuple`` with the following members (in order): + + * In instance of :class:`boto.sdb.domain.Domain` for the requested + domain + * The domain's name as a ``str`` + + :type domain_or_name: ``str`` or :class:`boto.sdb.domain.Domain` + :param domain_or_name: The domain or domain name to get the domain + and name for. + + :raises: :class:`boto.exception.SDBResponseError` when an invalid + domain name is specified. + + :rtype: tuple + :return: A ``tuple`` with contents outlined as per above. + """ + if (isinstance(domain_or_name, Domain)): + return (domain_or_name, domain_or_name.name) + else: + return (self.get_domain(domain_or_name), domain_or_name) + + def delete_domain(self, domain_or_name): + """ + Delete a SimpleDB domain. + + .. caution:: This will delete the domain and all items within the domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: bool + :return: True if successful + + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + return self.get_status('DeleteDomain', params) + + def domain_metadata(self, domain_or_name): + """ + Get the Metadata for a SimpleDB domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :rtype: :class:`boto.sdb.domain.DomainMetaData` object + :return: The newly created domain metadata object + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name} + d = self.get_object('DomainMetadata', params, DomainMetaData) + d.domain = domain + return d + + def put_attributes(self, domain_or_name, item_name, attributes, + replace=True, expected_value=None): + """ + Store attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being + stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type expected_value: list + :param expected_value: If supplied, this is a list or tuple consisting + of a single attribute name and expected value. The list can be + of the form: + + * ['name', 'value'] + + In which case the call will first verify that the attribute "name" + of this item has a value of "value". If it does, the delete + will proceed, otherwise a ConditionalCheckFailed error will be + returned. The list can also be of the form: + + * ['name', True|False] + + which will simply check for the existence (True) or + non-existence (False) of the attribute. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + self._build_name_value_list(params, attributes, replace) + if expected_value: + self._build_expected_value(params, expected_value) + return self.get_status('PutAttributes', params) + + def batch_put_attributes(self, domain_or_name, items, replace=True): + """ + Store attributes for multiple items in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name} + self._build_batch_list(params, items, replace) + return self.get_status('BatchPutAttributes', params, verb='POST') + + def get_attributes(self, domain_or_name, item_name, attribute_names=None, + consistent_read=False, item=None): + """ + Retrieve attributes for a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are + being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. + This parameter is optional. If not supplied, all attributes will + be retrieved for the item. + + :type consistent_read: bool + :param consistent_read: When set to true, ensures that the most recent + data is returned. + + :type item: :class:`boto.sdb.item.Item` + :keyword item: Instead of instantiating a new Item object, you may + specify one to update. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item with the requested attribute name/values set on it + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name, + 'ItemName' : item_name} + if consistent_read: + params['ConsistentRead'] = 'true' + if attribute_names: + if not isinstance(attribute_names, list): + attribute_names = [attribute_names] + self.build_list_params(params, attribute_names, 'AttributeName') + response = self.make_request('GetAttributes', params) + body = response.read() + if response.status == 200: + if item == None: + item = self.item_cls(domain, item_name) + h = handler.XmlHandler(item, self) + xml.sax.parseString(body, h) + return item + else: + raise SDBResponseError(response.status, response.reason, body) + + def delete_attributes(self, domain_or_name, item_name, attr_names=None, + expected_value=None): + """ + Delete attributes from a given item in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type item_name: string + :param item_name: The name of the item whose attributes are being + deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which + will cause all values associated with that attribute + name to be deleted or a dict or Item containing the + attribute names and keys and list of values to + delete as the value. If no value is supplied, + all attribute name/values for the item will be + deleted. + + :type expected_value: list + :param expected_value: If supplied, this is a list or tuple consisting + of a single attribute name and expected value. The list can be + of the form: + + * ['name', 'value'] + + In which case the call will first verify that the attribute "name" + of this item has a value of "value". If it does, the delete + will proceed, otherwise a ConditionalCheckFailed error will be + returned. The list can also be of the form: + + * ['name', True|False] + + which will simply check for the existence (True) or + non-existence (False) of the attribute. + + :rtype: bool + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName':domain_name, + 'ItemName' : item_name} + if attr_names: + if isinstance(attr_names, list): + self._build_name_list(params, attr_names) + elif isinstance(attr_names, dict) or isinstance(attr_names, self.item_cls): + self._build_name_value_list(params, attr_names) + if expected_value: + self._build_expected_value(params, expected_value) + return self.get_status('DeleteAttributes', params) + + def batch_delete_attributes(self, domain_or_name, items): + """ + Delete multiple items in a domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object. + :param domain_or_name: Either the name of a domain or a Domain object + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are either: + + * dictionaries of attribute names/values, exactly the + same as the attribute_names parameter of the scalar + put_attributes call. The attribute name/value pairs + will only be deleted if they match the name/value + pairs passed in. + * None which means that all attributes associated + with the item should be deleted. + + :return: True if successful + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'DomainName' : domain_name} + self._build_batch_list(params, items, False) + return self.get_status('BatchDeleteAttributes', params, verb='POST') + + def select(self, domain_or_name, query='', next_token=None, + consistent_read=False): + """ + Returns a set of Attributes for item names within domain_name that + match the query. The query must be expressed in using the SELECT + style syntax rather than the original SimpleDB query language. + Even though the select request does not require a domain object, + a domain object must be passed into this method so the Item objects + returned can point to the appropriate domain. + + :type domain_or_name: string or :class:`boto.sdb.domain.Domain` object + :param domain_or_name: Either the name of a domain or a Domain object + + :type query: string + :param query: The SimpleDB query to be performed. + + :type consistent_read: bool + :param consistent_read: When set to true, ensures that the most recent + data is returned. + + :rtype: ResultSet + :return: An iterator containing the results. + """ + domain, domain_name = self.get_domain_and_name(domain_or_name) + params = {'SelectExpression' : query} + if consistent_read: + params['ConsistentRead'] = 'true' + if next_token: + params['NextToken'] = next_token + try: + return self.get_list('Select', params, [('Item', self.item_cls)], + parent=domain) + except SDBResponseError, e: + e.body = "Query: %s\n%s" % (query, e.body) + raise e diff --git a/backup/src/boto/sdb/db/__init__.py b/backup/src/boto/sdb/db/__init__.py new file mode 100644 index 0000000..86044ed --- /dev/null +++ b/backup/src/boto/sdb/db/__init__.py @@ -0,0 +1,21 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + diff --git a/backup/src/boto/sdb/db/blob.py b/backup/src/boto/sdb/db/blob.py new file mode 100644 index 0000000..45a3624 --- /dev/null +++ b/backup/src/boto/sdb/db/blob.py @@ -0,0 +1,68 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +class Blob(object): + """Blob object""" + def __init__(self, value=None, file=None, id=None): + self._file = file + self.id = id + self.value = value + + @property + def file(self): + from StringIO import StringIO + if self._file: + f = self._file + else: + f = StringIO(self.value) + return f + + def __str__(self): + if hasattr(self.file, "get_contents_as_string"): + value = self.file.get_contents_as_string() + else: + value = self.file.getvalue() + try: + return str(value) + except: + return unicode(value) + + def read(self): + return self.file.read() + + def readline(self): + return self.file.readline() + + def next(self): + return self.file.next() + + def __iter__(self): + return iter(self.file) + + @property + def size(self): + if self._file: + return self._file.size + elif self.value: + return len(self.value) + else: + return 0 diff --git a/backup/src/boto/sdb/db/key.py b/backup/src/boto/sdb/db/key.py new file mode 100644 index 0000000..42a9d8d --- /dev/null +++ b/backup/src/boto/sdb/db/key.py @@ -0,0 +1,59 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Key(object): + + @classmethod + def from_path(cls, *args, **kwds): + raise NotImplementedError, "Paths are not currently supported" + + def __init__(self, encoded=None, obj=None): + self.name = None + if obj: + self.id = obj.id + self.kind = obj.kind() + else: + self.id = None + self.kind = None + + def app(self): + raise NotImplementedError, "Applications are not currently supported" + + def kind(self): + return self.kind + + def id(self): + return self.id + + def name(self): + raise NotImplementedError, "Key Names are not currently supported" + + def id_or_name(self): + return self.id + + def has_id_or_name(self): + return self.id != None + + def parent(self): + raise NotImplementedError, "Key parents are not currently supported" + + def __str__(self): + return self.id_or_name() diff --git a/backup/src/boto/sdb/db/manager/__init__.py b/backup/src/boto/sdb/db/manager/__init__.py new file mode 100644 index 0000000..0777796 --- /dev/null +++ b/backup/src/boto/sdb/db/manager/__init__.py @@ -0,0 +1,88 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto + +def get_manager(cls): + """ + Returns the appropriate Manager class for a given Model class. It does this by + looking in the boto config for a section like this:: + + [DB] + db_type = SimpleDB + db_user = + db_passwd = + db_name = my_domain + [DB_TestBasic] + db_type = SimpleDB + db_user = + db_passwd = + db_name = basic_domain + db_port = 1111 + + The values in the DB section are "generic values" that will be used if nothing more + specific is found. You can also create a section for a specific Model class that + gives the db info for that class. In the example above, TestBasic is a Model subclass. + """ + db_user = boto.config.get('DB', 'db_user', None) + db_passwd = boto.config.get('DB', 'db_passwd', None) + db_type = boto.config.get('DB', 'db_type', 'SimpleDB') + db_name = boto.config.get('DB', 'db_name', None) + db_table = boto.config.get('DB', 'db_table', None) + db_host = boto.config.get('DB', 'db_host', "sdb.amazonaws.com") + db_port = boto.config.getint('DB', 'db_port', 443) + enable_ssl = boto.config.getbool('DB', 'enable_ssl', True) + sql_dir = boto.config.get('DB', 'sql_dir', None) + debug = boto.config.getint('DB', 'debug', 0) + # first see if there is a fully qualified section name in the Boto config file + module_name = cls.__module__.replace('.', '_') + db_section = 'DB_' + module_name + '_' + cls.__name__ + if not boto.config.has_section(db_section): + db_section = 'DB_' + cls.__name__ + if boto.config.has_section(db_section): + db_user = boto.config.get(db_section, 'db_user', db_user) + db_passwd = boto.config.get(db_section, 'db_passwd', db_passwd) + db_type = boto.config.get(db_section, 'db_type', db_type) + db_name = boto.config.get(db_section, 'db_name', db_name) + db_table = boto.config.get(db_section, 'db_table', db_table) + db_host = boto.config.get(db_section, 'db_host', db_host) + db_port = boto.config.getint(db_section, 'db_port', db_port) + enable_ssl = boto.config.getint(db_section, 'enable_ssl', enable_ssl) + debug = boto.config.getint(db_section, 'debug', debug) + elif hasattr(cls.__bases__[0], "_manager"): + return cls.__bases__[0]._manager + if db_type == 'SimpleDB': + from sdbmanager import SDBManager + return SDBManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + elif db_type == 'PostgreSQL': + from pgmanager import PGManager + if db_table: + return PGManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + return None + elif db_type == 'XML': + from xmlmanager import XMLManager + return XMLManager(cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl) + else: + raise ValueError, 'Unknown db_type: %s' % db_type + diff --git a/backup/src/boto/sdb/db/manager/pgmanager.py b/backup/src/boto/sdb/db/manager/pgmanager.py new file mode 100644 index 0000000..73a93f0 --- /dev/null +++ b/backup/src/boto/sdb/db/manager/pgmanager.py @@ -0,0 +1,389 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +import psycopg2 +import psycopg2.extensions +import uuid +import os +import string +from boto.exception import SDBPersistenceError + +psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) + +class PGConverter: + + def __init__(self, manager): + self.manager = manager + self.type_map = {Key : (self.encode_reference, self.decode_reference), + Model : (self.encode_reference, self.decode_reference)} + + def encode(self, type, value): + if type in self.type_map: + encode = self.type_map[type][0] + return encode(value) + return value + + def decode(self, type, value): + if type in self.type_map: + decode = self.type_map[type][1] + return decode(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + s = "{" + new_value = [] + for v in value: + item_type = getattr(prop, 'item_type') + if Model in item_type.mro(): + item_type = Model + new_value.append('%s' % self.encode(item_type, v)) + s += ','.join(new_value) + s += "}" + return s + else: + return value + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if value != None: + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + if item_type != self.manager.cls: + return item_type._manager.decode_value(prop, value) + else: + item_type = Model + return [self.decode(item_type, v) for v in value] + return value + elif hasattr(prop, 'reference_class'): + ref_class = getattr(prop, 'reference_class') + if ref_class != self.manager.cls: + return ref_class._manager.decode_value(prop, value) + else: + return self.decode(prop.data_type, value) + elif hasattr(prop, 'calculated_type'): + calc_type = getattr(prop, 'calculated_type') + return self.decode(calc_type, value) + else: + return self.decode(prop.data_type, value) + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + + def decode_reference(self, value): + if not value: + return None + try: + return self.manager.get_object_from_id(value) + except: + raise ValueError, 'Unable to convert %s to Object' % value + +class PGManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, sql_dir, enable_ssl): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.sql_dir = sql_dir + self.in_transaction = False + self.converter = PGConverter(self) + self._connect() + + def _build_connect_string(self): + cs = 'dbname=%s user=%s password=%s host=%s port=%d' + return cs % (self.db_name, self.db_user, self.db_passwd, + self.db_host, self.db_port) + + def _connect(self): + self.connection = psycopg2.connect(self._build_connect_string()) + self.connection.set_client_encoding('UTF8') + self.cursor = self.connection.cursor() + + def _object_lister(self, cursor): + try: + for row in cursor: + yield self._object_from_row(row, cursor.description) + except StopIteration: + cursor.close() + raise StopIteration + + def _dict_from_row(self, row, description): + d = {} + for i in range(0, len(row)): + d[description[i][0]] = row[i] + return d + + def _object_from_row(self, row, description=None): + if not description: + description = self.cursor.description + d = self._dict_from_row(row, description) + obj = self.cls(d['id']) + obj._manager = self + obj._auto_update = False + for prop in obj.properties(hidden=False): + if prop.data_type != Key: + v = self.decode_value(prop, d[prop.name]) + v = prop.make_value_from_datastore(v) + if hasattr(prop, 'calculated_type'): + prop._set_direct(obj, v) + elif not prop.empty(v): + setattr(obj, prop.name, v) + else: + setattr(obj, prop.name, prop.default_value()) + return obj + + def _build_insert_qs(self, obj, calculated): + fields = [] + values = [] + templs = [] + id_calculated = [p for p in calculated if p.name == 'id'] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + fields.append('"%s"' % prop.name) + templs.append('%s') + qs = 'INSERT INTO "%s" (' % self.db_table + if len(id_calculated) == 0: + qs += '"id",' + qs += ','.join(fields) + qs += ") VALUES (" + if len(id_calculated) == 0: + qs += "'%s'," % obj.id + qs += ','.join(templs) + qs += ')' + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _build_update_qs(self, obj, calculated): + fields = [] + values = [] + for prop in obj.properties(hidden=False): + if prop not in calculated: + value = prop.get_value_for_datastore(obj) + if value != prop.default_value() or prop.required: + value = self.encode_value(prop, value) + values.append(value) + field = '"%s"=' % prop.name + field += '%s' + fields.append(field) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += ','.join(fields) + qs += """ WHERE "id" = '%s'""" % obj.id + if calculated: + qs += ' RETURNING ' + calc_values = ['"%s"' % p.name for p in calculated] + qs += ','.join(calc_values) + qs += ';' + return qs, values + + def _get_sql(self, mapping=None): + print '_get_sql' + sql = None + if self.sql_dir: + path = os.path.join(self.sql_dir, self.cls.__name__ + '.sql') + print path + if os.path.isfile(path): + fp = open(path) + sql = fp.read() + fp.close() + t = string.Template(sql) + sql = t.safe_substitute(mapping) + return sql + + def start_transaction(self): + print 'start_transaction' + self.in_transaction = True + + def end_transaction(self): + print 'end_transaction' + self.in_transaction = False + self.commit() + + def commit(self): + if not self.in_transaction: + print '!!commit on %s' % self.db_table + try: + self.connection.commit() + + except psycopg2.ProgrammingError, err: + self.connection.rollback() + raise err + + def rollback(self): + print '!!rollback on %s' % self.db_table + self.connection.rollback() + + def delete_table(self): + self.cursor.execute('DROP TABLE "%s";' % self.db_table) + self.commit() + + def create_table(self, mapping=None): + self.cursor.execute(self._get_sql(mapping)) + self.commit() + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def execute_sql(self, query): + self.cursor.execute(query, None) + self.commit() + + def query_sql(self, query, vars=None): + self.cursor.execute(query, vars) + return self.cursor.fetchall() + + def lookup(self, cls, name, value): + values = [] + qs = 'SELECT * FROM "%s" WHERE ' % self.db_table + found = False + for property in cls.properties(hidden=False): + if property.name == name: + found = True + value = self.encode_value(property, value) + values.append(value) + qs += "%s=" % name + qs += "%s" + if not found: + raise SDBPersistenceError('%s is not a valid field' % name) + qs += ';' + print qs + self.cursor.execute(qs, values) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + elif self.cursor.rowcount == 0: + raise KeyError, 'Object not found' + else: + raise LookupError, 'Multiple Objects Found' + + def query(self, cls, filters, limit=None, order_by=None): + parts = [] + qs = 'SELECT * FROM "%s"' % self.db_table + if filters: + qs += ' WHERE ' + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + value = self.encode_value(property, value) + parts.append(""""%s"%s'%s'""" % (name, op, value)) + if not found: + raise SDBPersistenceError('%s is not a valid field' % name) + qs += ','.join(parts) + qs += ';' + print qs + cursor = self.connection.cursor() + cursor.execute(qs) + return self._object_lister(cursor) + + def get_property(self, prop, obj, name): + qs = """SELECT "%s" FROM "%s" WHERE id='%s';""" % (name, self.db_table, obj.id) + print qs + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + rs = self.cursor.fetchone() + for prop in obj.properties(hidden=False): + if prop.name == name: + v = self.decode_value(prop, rs[0]) + return v + raise AttributeError, '%s not found' % name + + def set_property(self, prop, obj, name, value): + pass + value = self.encode_value(prop, value) + qs = 'UPDATE "%s" SET ' % self.db_table + qs += "%s='%s'" % (name, self.encode_value(prop, value)) + qs += " WHERE id='%s'" % obj.id + qs += ';' + print qs + self.cursor.execute(qs) + self.commit() + + def get_object(self, cls, id): + qs = """SELECT * FROM "%s" WHERE id='%s';""" % (self.db_table, id) + self.cursor.execute(qs, None) + if self.cursor.rowcount == 1: + row = self.cursor.fetchone() + return self._object_from_row(row, self.cursor.description) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + + def get_object_from_id(self, id): + return self.get_object(self.cls, id) + + def _find_calculated_props(self, obj): + return [p for p in obj.properties() if hasattr(p, 'calculated_type')] + + def save_object(self, obj): + obj._auto_update = False + calculated = self._find_calculated_props(obj) + if not obj.id: + obj.id = str(uuid.uuid4()) + qs, values = self._build_insert_qs(obj, calculated) + else: + qs, values = self._build_update_qs(obj, calculated) + print qs + self.cursor.execute(qs, values) + if calculated: + calc_values = self.cursor.fetchone() + print calculated + print calc_values + for i in range(0, len(calculated)): + prop = calculated[i] + prop._set_direct(obj, calc_values[i]) + self.commit() + + def delete_object(self, obj): + qs = """DELETE FROM "%s" WHERE id='%s';""" % (self.db_table, obj.id) + print qs + self.cursor.execute(qs) + self.commit() + + diff --git a/backup/src/boto/sdb/db/manager/sdbmanager.py b/backup/src/boto/sdb/db/manager/sdbmanager.py new file mode 100644 index 0000000..7dcc1c6 --- /dev/null +++ b/backup/src/boto/sdb/db/manager/sdbmanager.py @@ -0,0 +1,660 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +import re +from boto.utils import find_class +import uuid +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from boto.sdb.db.blob import Blob +from boto.sdb.db.property import ListProperty, MapProperty +from datetime import datetime, date, time +from boto.exception import SDBPersistenceError + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class TimeDecodeError(Exception): + pass + +class SDBConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + float : (self.encode_float, self.decode_float), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + datetime : (self.encode_datetime, self.decode_datetime), + date : (self.encode_date, self.decode_date), + time : (self.encode_time, self.decode_time), + Blob: (self.encode_blob, self.decode_blob), + } + + def encode(self, item_type, value): + try: + if Model in item_type.mro(): + item_type = Model + except: + pass + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + return value + + def encode_list(self, prop, value): + if value in (None, []): + return [] + if not isinstance(value, list): + # This is a little trick to avoid encoding when it's just a single value, + # since that most likely means it's from a query + item_type = getattr(prop, "item_type") + return self.encode(item_type, value) + # Just enumerate(value) won't work here because + # we need to add in some zero padding + # We support lists up to 1,000 attributes, since + # SDB technically only supports 1024 attributes anyway + values = {} + for k,v in enumerate(value): + values["%03d" % k] = v + return self.encode_map(prop, values) + + def encode_map(self, prop, value): + if value == None: + return None + if not isinstance(value, dict): + raise ValueError, 'Expected a dict value, got %s' % type(value) + new_value = [] + for key in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + encoded_value = self.encode(item_type, value[key]) + if encoded_value != None: + new_value.append('%s:%s' % (key, encoded_value)) + return new_value + + def encode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.encode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.encode_map(prop, value) + else: + return self.encode(prop.data_type, value) + + def decode_list(self, prop, value): + if not isinstance(value, list): + value = [value] + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + dec_val = {} + for val in value: + if val != None: + k,v = self.decode_map_element(item_type, val) + try: + k = int(k) + except: + k = v + dec_val[k] = v + value = dec_val.values() + return value + + def decode_map(self, prop, value): + if not isinstance(value, list): + value = [value] + ret_value = {} + item_type = getattr(prop, "item_type") + for val in value: + k,v = self.decode_map_element(item_type, val) + ret_value[k] = v + return ret_value + + def decode_map_element(self, item_type, value): + """Decode a single element for a map""" + key = value + if ":" in value: + key, value = value.split(':',1) + if Model in item_type.mro(): + value = item_type(id=value) + else: + value = self.decode(item_type, value) + return (key, value) + + def decode_prop(self, prop, value): + if isinstance(prop, ListProperty): + return self.decode_list(prop, value) + elif isinstance(prop, MapProperty): + return self.decode_map(prop, value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + value += 2147483648 + return '%010d' % value + + def decode_int(self, value): + try: + value = int(value) + except: + boto.log.error("Error, %s is not an integer" % value) + value = 0 + value = int(value) + value -= 2147483648 + return int(value) + + def encode_long(self, value): + value = long(value) + value += 9223372036854775808 + return '%020d' % value + + def decode_long(self, value): + value = long(value) + value -= 9223372036854775808 + return value + + def encode_bool(self, value): + if value == True or str(value).lower() in ("true", "yes"): + return 'true' + else: + return 'false' + + def decode_bool(self, value): + if value.lower() == 'true': + return True + else: + return False + + def encode_float(self, value): + """ + See http://tools.ietf.org/html/draft-wood-ldapext-float-00. + """ + s = '%e' % value + l = s.split('e') + mantissa = l[0].ljust(18, '0') + exponent = l[1] + if value == 0.0: + case = '3' + exponent = '000' + elif mantissa[0] != '-' and exponent[0] == '+': + case = '5' + exponent = exponent[1:].rjust(3, '0') + elif mantissa[0] != '-' and exponent[0] == '-': + case = '4' + exponent = 999 + int(exponent) + exponent = '%03d' % exponent + elif mantissa[0] == '-' and exponent[0] == '-': + case = '2' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = exponent[1:].rjust(3, '0') + else: + case = '1' + mantissa = '%f' % (10 + float(mantissa)) + mantissa = mantissa.ljust(18, '0') + exponent = 999 - int(exponent) + exponent = '%03d' % exponent + return '%s %s %s' % (case, exponent, mantissa) + + def decode_float(self, value): + case = value[0] + exponent = value[2:5] + mantissa = value[6:] + if case == '3': + return 0.0 + elif case == '5': + pass + elif case == '4': + exponent = '%03d' % (int(exponent) - 999) + elif case == '2': + mantissa = '%f' % (float(mantissa) - 10) + exponent = '-' + exponent + else: + mantissa = '%f' % (float(mantissa) - 10) + exponent = '%03d' % abs((int(exponent) - 999)) + return float(mantissa + 'e' + exponent) + + def encode_datetime(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + return value.strftime(ISO8601) + + def decode_datetime(self, value): + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_date(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + return value.isoformat() + + def decode_date(self, value): + try: + value = value.split("-") + return date(int(value[0]), int(value[1]), int(value[2])) + except: + return None + + encode_time = encode_date + + def decode_time(self, value): + """ converts strings in the form of HH:MM:SS.mmmmmm + (created by datetime.time.isoformat()) to + datetime.time objects. + + Timzone-aware strings ("HH:MM:SS.mmmmmm+HH:MM") won't + be handled right now and will raise TimeDecodeError. + """ + if '-' in value or '+' in value: + # TODO: Handle tzinfo + raise TimeDecodeError("Can't handle timezone aware objects: %r" % value) + tmp = value.split('.') + arg = map(int, tmp[0].split(':')) + if len(tmp) == 2: + arg.append(int(tmp[1])) + return time(*arg) + + def encode_reference(self, value): + if value in (None, 'None', '', ' '): + return None + if isinstance(value, str) or isinstance(value, unicode): + return value + else: + return value.id + + def decode_reference(self, value): + if not value or value == "None": + return None + return value + + def encode_blob(self, value): + if not value: + return None + if isinstance(value, str): + return value + + if not value.id: + bucket = self.manager.get_blob_bucket() + key = bucket.new_key(str(uuid.uuid4())) + value.id = "s3://%s/%s" % (key.bucket.name, key.name) + else: + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value.id) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + raise SDBPersistenceError("Invalid Blob ID: %s" % value.id) + + if value.value != None: + key.set_contents_from_string(value.value) + return value.id + + + def decode_blob(self, value): + if not value: + return None + match = re.match("^s3:\/\/([^\/]*)\/(.*)$", value) + if match: + s3 = self.manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + key = bucket.get_key(match.group(2)) + else: + return None + if key: + return Blob(file=key, id="s3://%s/%s" % (key.bucket.name, key.name)) + else: + return None + +class SDBManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl, consistent=None): + self.cls = cls + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.enable_ssl = enable_ssl + self.s3 = None + self.bucket = None + self.converter = SDBConverter(self) + self._sdb = None + self._domain = None + if consistent == None and hasattr(cls, "__consistent__"): + consistent = cls.__consistent__ + self.consistent = consistent + + @property + def sdb(self): + if self._sdb is None: + self._connect() + return self._sdb + + @property + def domain(self): + if self._domain is None: + self._connect() + return self._domain + + def _connect(self): + self._sdb = boto.connect_sdb(aws_access_key_id=self.db_user, + aws_secret_access_key=self.db_passwd, + is_secure=self.enable_ssl) + # This assumes that the domain has already been created + # It's much more efficient to do it this way rather than + # having this make a roundtrip each time to validate. + # The downside is that if the domain doesn't exist, it breaks + self._domain = self._sdb.lookup(self.db_name, validate=False) + if not self._domain: + self._domain = self._sdb.create_domain(self.db_name) + + def _object_lister(self, cls, query_lister): + for item in query_lister: + obj = self.get_object(cls, item.name, item) + if obj: + yield obj + + def encode_value(self, prop, value): + if value == None: + return None + if not prop: + return str(value) + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.db_user, self.db_passwd) + return self.s3 + + def get_blob_bucket(self, bucket_name=None): + s3 = self.get_s3_connection() + bucket_name = "%s-%s" % (s3.aws_access_key_id, self.domain.name) + bucket_name = bucket_name.lower() + try: + self.bucket = s3.get_bucket(bucket_name) + except: + self.bucket = s3.create_bucket(bucket_name) + return self.bucket + + def load_object(self, obj): + if not obj._loaded: + a = self.domain.get_attributes(obj.id,consistent_read=self.consistent) + if a.has_key('__type__'): + for prop in obj.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + try: + setattr(obj, prop.name, value) + except Exception, e: + boto.log.exception(e) + obj._loaded = True + + def get_object(self, cls, id, a=None): + obj = None + if not a: + a = self.domain.get_attributes(id,consistent_read=self.consistent) + if a.has_key('__type__'): + if not cls or a['__type__'] != cls.__name__: + cls = find_class(a['__module__'], a['__type__']) + if cls: + params = {} + for prop in cls.properties(hidden=False): + if a.has_key(prop.name): + value = self.decode_value(prop, a[prop.name]) + value = prop.make_value_from_datastore(value) + params[prop.name] = value + obj = cls(id, **params) + obj._loaded = True + else: + s = '(%s) class %s.%s not found' % (id, a['__module__'], a['__type__']) + boto.log.info('sdbmanager: %s' % s) + return obj + + def get_object_from_id(self, id): + return self.get_object(None, id) + + def query(self, query): + query_str = "select * from `%s` %s" % (self.domain.name, self._build_filter_part(query.model_class, query.filters, query.sort_by, query.select)) + if query.limit: + query_str += " limit %s" % query.limit + rs = self.domain.select(query_str, max_items=query.limit, next_token = query.next_token) + query.rs = rs + return self._object_lister(query.model_class, rs) + + def count(self, cls, filters, quick=True, sort_by=None, select=None): + """ + Get the number of results that would + be returned in this query + """ + query = "select count(*) from `%s` %s" % (self.domain.name, self._build_filter_part(cls, filters, sort_by, select)) + count = 0 + for row in self.domain.select(query): + count += int(row['Count']) + if quick: + return count + return count + + + def _build_filter(self, property, name, op, val): + if name == "__id__": + name = 'itemName()' + if name != "itemName()": + name = '`%s`' % name + if val == None: + if op in ('is','='): + return "%(name)s is null" % {"name": name} + elif op in ('is not', '!='): + return "%s is not null" % name + else: + val = "" + if property.__class__ == ListProperty: + if op in ("is", "="): + op = "like" + elif op in ("!=", "not"): + op = "not like" + if not(op in ["like", "not like"] and val.startswith("%")): + val = "%%:%s" % val + return "%s %s '%s'" % (name, op, val.replace("'", "''")) + + def _build_filter_part(self, cls, filters, order_by=None, select=None): + """ + Build the filter part + """ + import types + query_parts = [] + order_by_filtered = False + if order_by: + if order_by[0] == "-": + order_by_method = "DESC"; + order_by = order_by[1:] + else: + order_by_method = "ASC"; + if isinstance(filters, str) or isinstance(filters, unicode): + query = "WHERE `__type__` = '%s' AND %s" % (cls.__name__, filters) + if order_by != None: + query += " ORDER BY `%s` %s" % (order_by, order_by_method) + return query + + for filter in filters: + filter_parts = [] + filter_props = filter[0] + if type(filter_props) != list: + filter_props = [filter_props] + for filter_prop in filter_props: + (name, op) = filter_prop.strip().split(" ", 1) + value = filter[1] + property = cls.find_property(name) + if name == order_by: + order_by_filtered = True + if types.TypeType(value) == types.ListType: + filter_parts_sub = [] + for val in value: + val = self.encode_value(property, val) + if isinstance(val, list): + for v in val: + filter_parts_sub.append(self._build_filter(property, name, op, v)) + else: + filter_parts_sub.append(self._build_filter(property, name, op, val)) + filter_parts.append("(%s)" % (" OR ".join(filter_parts_sub))) + else: + val = self.encode_value(property, value) + if isinstance(val, list): + for v in val: + filter_parts.append(self._build_filter(property, name, op, v)) + else: + filter_parts.append(self._build_filter(property, name, op, val)) + query_parts.append("(%s)" % (" or ".join(filter_parts))) + + + type_query = "(`__type__` = '%s'" % cls.__name__ + for subclass in self._get_all_decendents(cls).keys(): + type_query += " or `__type__` = '%s'" % subclass + type_query +=")" + query_parts.append(type_query) + + order_by_query = "" + if order_by: + if not order_by_filtered: + query_parts.append("`%s` LIKE '%%'" % order_by) + order_by_query = " ORDER BY `%s` %s" % (order_by, order_by_method) + + if select: + query_parts.append("(%s)" % select) + + if len(query_parts) > 0: + return "WHERE %s %s" % (" AND ".join(query_parts), order_by_query) + else: + return "" + + + def _get_all_decendents(self, cls): + """Get all decendents for a given class""" + decendents = {} + for sc in cls.__sub_classes__: + decendents[sc.__name__] = sc + decendents.update(self._get_all_decendents(sc)) + return decendents + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in SimpleDB" + + def save_object(self, obj): + if not obj.id: + obj.id = str(uuid.uuid4()) + + attrs = {'__type__' : obj.__class__.__name__, + '__module__' : obj.__class__.__module__, + '__lineage__' : obj.get_lineage()} + del_attrs = [] + for property in obj.properties(hidden=False): + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + if value == []: + value = None + if value == None: + del_attrs.append(property.name) + continue + attrs[property.name] = value + if property.unique: + try: + args = {property.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % property.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, attrs, replace=True) + if len(del_attrs) > 0: + self.domain.delete_attributes(obj.id, del_attrs) + return obj + + def delete_object(self, obj): + self.domain.delete_attributes(obj.id) + + def set_property(self, prop, obj, name, value): + value = prop.get_value_for_datastore(obj) + value = self.encode_value(prop, value) + if prop.unique: + try: + args = {prop.name: value} + obj2 = obj.find(**args).next() + if obj2.id != obj.id: + raise SDBPersistenceError("Error: %s must be unique!" % prop.name) + except(StopIteration): + pass + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def get_property(self, prop, obj, name): + a = self.domain.get_attributes(obj.id,consistent_read=self.consistent) + + # try to get the attribute value from SDB + if name in a: + value = self.decode_value(prop, a[name]) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + return value + raise AttributeError, '%s not found' % name + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name,consistent_read=self.consistent) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + diff --git a/backup/src/boto/sdb/db/manager/xmlmanager.py b/backup/src/boto/sdb/db/manager/xmlmanager.py new file mode 100644 index 0000000..9765df1 --- /dev/null +++ b/backup/src/boto/sdb/db/manager/xmlmanager.py @@ -0,0 +1,517 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +import boto +from boto.utils import find_class, Password +from boto.sdb.db.key import Key +from boto.sdb.db.model import Model +from datetime import datetime +from xml.dom.minidom import getDOMImplementation, parse, parseString, Node + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class XMLConverter: + """ + Responsible for converting base Python types to format compatible with underlying + database. For SimpleDB, that means everything needs to be converted to a string + when stored in SimpleDB and from a string when retrieved. + + To convert a value, pass it to the encode or decode method. The encode method + will take a Python native value and convert to DB format. The decode method will + take a DB format value and convert it to Python native format. To find the appropriate + method to call, the generic encode/decode methods will look for the type-specific + method by searching for a method called "encode_" or "decode_". + """ + def __init__(self, manager): + self.manager = manager + self.type_map = { bool : (self.encode_bool, self.decode_bool), + int : (self.encode_int, self.decode_int), + long : (self.encode_long, self.decode_long), + Model : (self.encode_reference, self.decode_reference), + Key : (self.encode_reference, self.decode_reference), + Password : (self.encode_password, self.decode_password), + datetime : (self.encode_datetime, self.decode_datetime)} + + def get_text_value(self, parent_node): + value = '' + for node in parent_node.childNodes: + if node.nodeType == node.TEXT_NODE: + value += node.data + return value + + def encode(self, item_type, value): + if item_type in self.type_map: + encode = self.type_map[item_type][0] + return encode(value) + return value + + def decode(self, item_type, value): + if item_type in self.type_map: + decode = self.type_map[item_type][1] + return decode(value) + else: + value = self.get_text_value(value) + return value + + def encode_prop(self, prop, value): + if isinstance(value, list): + if hasattr(prop, 'item_type'): + new_value = [] + for v in value: + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + new_value.append(self.encode(item_type, v)) + return new_value + else: + return value + else: + return self.encode(prop.data_type, value) + + def decode_prop(self, prop, value): + if prop.data_type == list: + if hasattr(prop, 'item_type'): + item_type = getattr(prop, "item_type") + if Model in item_type.mro(): + item_type = Model + values = [] + for item_node in value.getElementsByTagName('item'): + value = self.decode(item_type, item_node) + values.append(value) + return values + else: + return self.get_text_value(value) + else: + return self.decode(prop.data_type, value) + + def encode_int(self, value): + value = int(value) + return '%d' % value + + def decode_int(self, value): + value = self.get_text_value(value) + if value: + value = int(value) + else: + value = None + return value + + def encode_long(self, value): + value = long(value) + return '%d' % value + + def decode_long(self, value): + value = self.get_text_value(value) + return long(value) + + def encode_bool(self, value): + if value == True: + return 'true' + else: + return 'false' + + def decode_bool(self, value): + value = self.get_text_value(value) + if value.lower() == 'true': + return True + else: + return False + + def encode_datetime(self, value): + return value.strftime(ISO8601) + + def decode_datetime(self, value): + value = self.get_text_value(value) + try: + return datetime.strptime(value, ISO8601) + except: + return None + + def encode_reference(self, value): + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + val_node = self.manager.doc.createElement("object") + val_node.setAttribute('id', value.id) + val_node.setAttribute('class', '%s.%s' % (value.__class__.__module__, value.__class__.__name__)) + return val_node + + def decode_reference(self, value): + if not value: + return None + try: + value = value.childNodes[0] + class_name = value.getAttribute("class") + id = value.getAttribute("id") + cls = find_class(class_name) + return cls.get_by_ids(id) + except: + return None + + def encode_password(self, value): + if value and len(value) > 0: + return str(value) + else: + return None + + def decode_password(self, value): + value = self.get_text_value(value) + return Password(value) + + +class XMLManager(object): + + def __init__(self, cls, db_name, db_user, db_passwd, + db_host, db_port, db_table, ddl_dir, enable_ssl): + self.cls = cls + if not db_name: + db_name = cls.__name__.lower() + self.db_name = db_name + self.db_user = db_user + self.db_passwd = db_passwd + self.db_host = db_host + self.db_port = db_port + self.db_table = db_table + self.ddl_dir = ddl_dir + self.s3 = None + self.converter = XMLConverter(self) + self.impl = getDOMImplementation() + self.doc = self.impl.createDocument(None, 'objects', None) + + self.connection = None + self.enable_ssl = enable_ssl + self.auth_header = None + if self.db_user: + import base64 + base64string = base64.encodestring('%s:%s' % (self.db_user, self.db_passwd))[:-1] + authheader = "Basic %s" % base64string + self.auth_header = authheader + + def _connect(self): + if self.db_host: + if self.enable_ssl: + from httplib import HTTPSConnection as Connection + else: + from httplib import HTTPConnection as Connection + + self.connection = Connection(self.db_host, self.db_port) + + def _make_request(self, method, url, post_data=None, body=None): + """ + Make a request on this connection + """ + if not self.connection: + self._connect() + try: + self.connection.close() + except: + pass + self.connection.connect() + headers = {} + if self.auth_header: + headers["Authorization"] = self.auth_header + self.connection.request(method, url, body, headers) + resp = self.connection.getresponse() + return resp + + def new_doc(self): + return self.impl.createDocument(None, 'objects', None) + + def _object_lister(self, cls, doc): + for obj_node in doc.getElementsByTagName('object'): + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + if prop: + if hasattr(prop, 'item_type'): + value = self.get_list(prop_node, prop.item_type) + else: + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + setattr(obj, prop.name, value) + yield obj + + def reset(self): + self._connect() + + def get_doc(self): + return self.doc + + def encode_value(self, prop, value): + return self.converter.encode_prop(prop, value) + + def decode_value(self, prop, value): + return self.converter.decode_prop(prop, value) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + + def get_list(self, prop_node, item_type): + values = [] + try: + items_node = prop_node.getElementsByTagName('items')[0] + except: + return [] + for item_node in items_node.getElementsByTagName('item'): + value = self.converter.decode(item_type, item_node) + values.append(value) + return values + + def get_object_from_doc(self, cls, id, doc): + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + obj = cls(id) + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = obj.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + try: + setattr(obj, prop.name, value) + except: + pass + return obj + + def get_props_from_doc(self, cls, id, doc): + """ + Pull out the properties from this document + Returns the class, the properties in a hash, and the id if provided as a tuple + :return: (cls, props, id) + """ + obj_node = doc.getElementsByTagName('object')[0] + if not cls: + class_name = obj_node.getAttribute('class') + cls = find_class(class_name) + if not id: + id = obj_node.getAttribute('id') + props = {} + for prop_node in obj_node.getElementsByTagName('property'): + prop_name = prop_node.getAttribute('name') + prop = cls.find_property(prop_name) + value = self.decode_value(prop, prop_node) + value = prop.make_value_from_datastore(value) + if value != None: + props[prop.name] = value + return (cls, props, id) + + + def get_object(self, cls, id): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + url = "/%s/%s" % (self.db_name, id) + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self.get_object_from_doc(cls, id, doc) + + def query(self, cls, filters, limit=None, order_by=None): + if not self.connection: + self._connect() + + if not self.connection: + raise NotImplementedError("Can't query without a database connection") + + from urllib import urlencode + + query = str(self._build_query(cls, filters, limit, order_by)) + if query: + url = "/%s?%s" % (self.db_name, urlencode({"query": query})) + else: + url = "/%s" % self.db_name + resp = self._make_request('GET', url) + if resp.status == 200: + doc = parse(resp) + else: + raise Exception("Error: %s" % resp.status) + return self._object_lister(cls, doc) + + def _build_query(self, cls, filters, limit, order_by): + import types + if len(filters) > 4: + raise Exception('Too many filters, max is 4') + parts = [] + properties = cls.properties(hidden=False) + for filter, value in filters: + name, op = filter.strip().split() + found = False + for property in properties: + if property.name == name: + found = True + if types.TypeType(value) == types.ListType: + filter_parts = [] + for val in value: + val = self.encode_value(property, val) + filter_parts.append("'%s' %s '%s'" % (name, op, val)) + parts.append("[%s]" % " OR ".join(filter_parts)) + else: + value = self.encode_value(property, value) + parts.append("['%s' %s '%s']" % (name, op, value)) + if not found: + raise Exception('%s is not a valid field' % name) + if order_by: + if order_by.startswith("-"): + key = order_by[1:] + type = "desc" + else: + key = order_by + type = "asc" + parts.append("['%s' starts-with ''] sort '%s' %s" % (key, key, type)) + return ' intersection '.join(parts) + + def query_gql(self, query_string, *args, **kwds): + raise NotImplementedError, "GQL queries not supported in XML" + + def save_list(self, doc, items, prop_node): + items_node = doc.createElement('items') + prop_node.appendChild(items_node) + for item in items: + item_node = doc.createElement('item') + items_node.appendChild(item_node) + if isinstance(item, Node): + item_node.appendChild(item) + else: + text_node = doc.createTextNode(item) + item_node.appendChild(text_node) + + def save_object(self, obj): + """ + Marshal the object and do a PUT + """ + doc = self.marshal_object(obj) + if obj.id: + url = "/%s/%s" % (self.db_name, obj.id) + else: + url = "/%s" % (self.db_name) + resp = self._make_request("PUT", url, body=doc.toxml()) + new_obj = self.get_object_from_doc(obj.__class__, None, parse(resp)) + obj.id = new_obj.id + for prop in obj.properties(): + try: + propname = prop.name + except AttributeError: + propname = None + if propname: + value = getattr(new_obj, prop.name) + if value: + setattr(obj, prop.name, value) + return obj + + + def marshal_object(self, obj, doc=None): + if not doc: + doc = self.new_doc() + if not doc: + doc = self.doc + obj_node = doc.createElement('object') + + if obj.id: + obj_node.setAttribute('id', obj.id) + + obj_node.setAttribute('class', '%s.%s' % (obj.__class__.__module__, + obj.__class__.__name__)) + root = doc.documentElement + root.appendChild(obj_node) + for property in obj.properties(hidden=False): + prop_node = doc.createElement('property') + prop_node.setAttribute('name', property.name) + prop_node.setAttribute('type', property.type_name) + value = property.get_value_for_datastore(obj) + if value is not None: + value = self.encode_value(property, value) + if isinstance(value, list): + self.save_list(doc, value, prop_node) + elif isinstance(value, Node): + prop_node.appendChild(value) + else: + text_node = doc.createTextNode(unicode(value).encode("ascii", "ignore")) + prop_node.appendChild(text_node) + obj_node.appendChild(prop_node) + + return doc + + def unmarshal_object(self, fp, cls=None, id=None): + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_object_from_doc(cls, id, doc) + + def unmarshal_props(self, fp, cls=None, id=None): + """ + Same as unmarshalling an object, except it returns + from "get_props_from_doc" + """ + if isinstance(fp, str) or isinstance(fp, unicode): + doc = parseString(fp) + else: + doc = parse(fp) + return self.get_props_from_doc(cls, id, doc) + + def delete_object(self, obj): + url = "/%s/%s" % (self.db_name, obj.id) + return self._make_request("DELETE", url) + + def set_key_value(self, obj, name, value): + self.domain.put_attributes(obj.id, {name : value}, replace=True) + + def delete_key_value(self, obj, name): + self.domain.delete_attributes(obj.id, name) + + def get_key_value(self, obj, name): + a = self.domain.get_attributes(obj.id, name) + if a.has_key(name): + return a[name] + else: + return None + + def get_raw_item(self, obj): + return self.domain.get_item(obj.id) + + def set_property(self, prop, obj, name, value): + pass + + def get_property(self, prop, obj, name): + pass + + def load_object(self, obj): + if not obj._loaded: + obj = obj.get_by_id(obj.id) + obj._loaded = True + return obj + diff --git a/backup/src/boto/sdb/db/model.py b/backup/src/boto/sdb/db/model.py new file mode 100644 index 0000000..18bec4b --- /dev/null +++ b/backup/src/boto/sdb/db/model.py @@ -0,0 +1,248 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sdb.db.manager import get_manager +from boto.sdb.db.property import Property +from boto.sdb.db.key import Key +from boto.sdb.db.query import Query +import boto + +class ModelMeta(type): + "Metaclass for all Models" + + def __init__(cls, name, bases, dict): + super(ModelMeta, cls).__init__(name, bases, dict) + # Make sure this is a subclass of Model - mainly copied from django ModelBase (thanks!) + cls.__sub_classes__ = [] + try: + if filter(lambda b: issubclass(b, Model), bases): + for base in bases: + base.__sub_classes__.append(cls) + cls._manager = get_manager(cls) + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.__property_config__(cls, key) + prop_names = [] + props = cls.properties() + for prop in props: + if not prop.__class__.__name__.startswith('_'): + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'Model' isn't defined yet, meaning we're looking at our own + # Model class, defined below. + pass + +class Model(object): + __metaclass__ = ModelMeta + __consistent__ = False # Consistent is set off by default + id = None + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def kind(cls): + return cls.__name__ + + @classmethod + def _get_by_id(cls, id, manager=None): + if not manager: + manager = cls._manager + return manager.get_object(cls, id) + + @classmethod + def get_by_id(cls, ids=None, parent=None): + if isinstance(ids, list): + objs = [cls._get_by_id(id) for id in ids] + return objs + else: + return cls._get_by_id(ids) + + get_by_ids = get_by_id + + @classmethod + def get_by_key_name(cls, key_names, parent=None): + raise NotImplementedError, "Key Names are not currently supported" + + @classmethod + def find(cls, limit=None, next_token=None, **params): + q = Query(cls, limit=limit, next_token=next_token) + for key, value in params.items(): + q.filter('%s =' % key, value) + return q + + @classmethod + def all(cls, limit=None, next_token=None): + return cls.find(limit=limit, next_token=next_token) + + @classmethod + def get_or_insert(key_name, **kw): + raise NotImplementedError, "get_or_insert not currently supported" + + @classmethod + def properties(cls, hidden=True): + properties = [] + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if hidden or not prop.__class__.__name__.startswith('_'): + properties.append(prop) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + @classmethod + def find_property(cls, prop_name): + property = None + while cls: + for key in cls.__dict__.keys(): + prop = cls.__dict__[key] + if isinstance(prop, Property): + if not prop.__class__.__name__.startswith('_') and prop_name == prop.name: + property = prop + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return property + + @classmethod + def get_xmlmanager(cls): + if not hasattr(cls, '_xmlmanager'): + from boto.sdb.db.manager.xmlmanager import XMLManager + cls._xmlmanager = XMLManager(cls, None, None, None, + None, None, None, None, False) + return cls._xmlmanager + + @classmethod + def from_xml(cls, fp): + xmlmanager = cls.get_xmlmanager() + return xmlmanager.unmarshal_object(fp) + + def __init__(self, id=None, **kw): + self._loaded = False + # first try to initialize all properties to their default values + for prop in self.properties(hidden=False): + try: + setattr(self, prop.name, prop.default_value()) + except ValueError: + pass + if kw.has_key('manager'): + self._manager = kw['manager'] + self.id = id + for key in kw: + if key != 'manager': + # We don't want any errors populating up when loading an object, + # so if it fails we just revert to it's default value + try: + setattr(self, key, kw[key]) + except Exception, e: + boto.log.exception(e) + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def __str__(self): + return str(self.id) + + def __eq__(self, other): + return other and isinstance(other, Model) and self.id == other.id + + def _get_raw_item(self): + return self._manager.get_raw_item(self) + + def load(self): + if self.id and not self._loaded: + self._manager.load_object(self) + + def reload(self): + if self.id: + self._loaded = False + self._manager.load_object(self) + + def put(self): + self._manager.save_object(self) + + save = put + + def delete(self): + self._manager.delete_object(self) + + def key(self): + return Key(obj=self) + + def set_manager(self, manager): + self._manager = manager + + def to_dict(self): + props = {} + for prop in self.properties(hidden=False): + props[prop.name] = getattr(self, prop.name) + obj = {'properties' : props, + 'id' : self.id} + return {self.__class__.__name__ : obj} + + def to_xml(self, doc=None): + xmlmanager = self.get_xmlmanager() + doc = xmlmanager.marshal_object(self, doc) + return doc + + @classmethod + def find_subclass(cls, name): + """Find a subclass with a given name""" + if name == cls.__name__: + return cls + for sc in cls.__sub_classes__: + r = sc.find_subclass(name) + if r != None: + return r + +class Expando(Model): + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._manager.set_key_value(self, name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + value = self._manager.get_key_value(self, name) + if value: + object.__setattr__(self, name, value) + return value + raise AttributeError + + diff --git a/backup/src/boto/sdb/db/property.py b/backup/src/boto/sdb/db/property.py new file mode 100644 index 0000000..6db7cac --- /dev/null +++ b/backup/src/boto/sdb/db/property.py @@ -0,0 +1,624 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import datetime +from key import Key +from boto.utils import Password +from boto.sdb.db.query import Query +import re +import boto +import boto.s3.key +from boto.sdb.db.blob import Blob + +class Property(object): + + data_type = str + type_name = '' + name = '' + verbose_name = '' + + def __init__(self, verbose_name=None, name=None, default=None, required=False, + validator=None, choices=None, unique=False): + self.verbose_name = verbose_name + self.name = name + self.default = default + self.required = required + self.validator = validator + self.choices = choices + if self.name: + self.slot_name = '_' + self.name + else: + self.slot_name = '_' + self.unique = unique + + def __get__(self, obj, objtype): + if obj: + obj.load() + return getattr(obj, self.slot_name) + else: + return None + + def __set__(self, obj, value): + self.validate(value) + + # Fire off any on_set functions + try: + if obj._loaded and hasattr(obj, "on_set_%s" % self.name): + fnc = getattr(obj, "on_set_%s" % self.name) + value = fnc(value) + except Exception: + boto.log.exception("Exception running on_set_%s" % self.name) + + setattr(obj, self.slot_name, value) + + def __property_config__(self, model_class, property_name): + self.model_class = model_class + self.name = property_name + self.slot_name = '_' + self.name + + def default_validator(self, value): + if value == self.default_value(): + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def default_value(self): + return self.default + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if self.choices and value and not value in self.choices: + raise ValueError, '%s not a valid choice for %s.%s' % (value, self.model_class.__name__, self.name) + if self.validator: + self.validator(value) + else: + self.default_validator(value) + return value + + def empty(self, value): + return not value + + def get_value_for_datastore(self, model_instance): + return getattr(model_instance, self.name) + + def make_value_from_datastore(self, value): + return value + + def get_choices(self): + if callable(self.choices): + return self.choices() + return self.choices + +def validate_string(value): + if value == None: + return + elif isinstance(value, str) or isinstance(value, unicode): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class StringProperty(Property): + + type_name = 'String' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=validate_string, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + +class TextProperty(Property): + + type_name = 'Text' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False, max_length=None): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max_length = max_length + + def validate(self, value): + if not isinstance(value, str) and not isinstance(value, unicode): + raise TypeError, 'Expecting Text, got %s' % type(value) + if self.max_length and len(value) > self.max_length: + raise ValueError, 'Length of value greater than maxlength %s' % self.max_length + +class PasswordProperty(StringProperty): + """ + Hashed property who's original value can not be + retrieved, but still can be compaired. + """ + data_type = Password + type_name = 'Password' + + def __init__(self, verbose_name=None, name=None, default='', required=False, + validator=None, choices=None, unique=False): + StringProperty.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def make_value_from_datastore(self, value): + p = Password(value) + return p + + def get_value_for_datastore(self, model_instance): + value = StringProperty.get_value_for_datastore(self, model_instance) + if value and len(value): + return str(value) + else: + return None + + def __set__(self, obj, value): + if not isinstance(value, Password): + p = Password() + p.set(value) + value = p + Property.__set__(self, obj, value) + + def __get__(self, obj, objtype): + return Password(StringProperty.__get__(self, obj, objtype)) + + def validate(self, value): + value = Property.validate(self, value) + if isinstance(value, Password): + if len(value) > 1024: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting Password, got %s' % type(value) + +class BlobProperty(Property): + data_type = Blob + type_name = "blob" + + def __set__(self, obj, value): + if value != self.default_value(): + if not isinstance(value, Blob): + oldb = self.__get__(obj, type(obj)) + id = None + if oldb: + id = oldb.id + b = Blob(value=value, id=id) + value = b + Property.__set__(self, obj, value) + +class S3KeyProperty(Property): + + data_type = boto.s3.key.Key + type_name = 'S3Key' + validate_regex = "^s3:\/\/([^\/]*)\/(.*)$" + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + + def validate(self, value): + if value == self.default_value() or value == str(self.default_value()): + return self.default_value() + if isinstance(value, self.data_type): + return + match = re.match(self.validate_regex, value) + if match: + return + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def __get__(self, obj, objtype): + value = Property.__get__(self, obj, objtype) + if value: + if isinstance(value, self.data_type): + return value + match = re.match(self.validate_regex, value) + if match: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(match.group(1), validate=False) + k = bucket.get_key(match.group(2)) + if not k: + k = bucket.new_key(match.group(2)) + k.set_contents_from_string("") + return k + else: + return value + + def get_value_for_datastore(self, model_instance): + value = Property.get_value_for_datastore(self, model_instance) + if value: + return "s3://%s/%s" % (value.bucket.name, value.name) + else: + return None + +class IntegerProperty(Property): + + data_type = int + type_name = 'Integer' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False, max=2147483647, min=-2147483648): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.max = max + self.min = min + + def validate(self, value): + value = int(value) + value = Property.validate(self, value) + if value > self.max: + raise ValueError, 'Maximum value is %d' % self.max + if value < self.min: + raise ValueError, 'Minimum value is %d' % self.min + return value + + def empty(self, value): + return value is None + + def __set__(self, obj, value): + if value == "" or value == None: + value = 0 + return Property.__set__(self, obj, value) + + + +class LongProperty(Property): + + data_type = long + type_name = 'Long' + + def __init__(self, verbose_name=None, name=None, default=0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = long(value) + value = Property.validate(self, value) + min = -9223372036854775808 + max = 9223372036854775807 + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + return value + + def empty(self, value): + return value is None + +class BooleanProperty(Property): + + data_type = bool + type_name = 'Boolean' + + def __init__(self, verbose_name=None, name=None, default=False, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def empty(self, value): + return value is None + +class FloatProperty(Property): + + data_type = float + type_name = 'Float' + + def __init__(self, verbose_name=None, name=None, default=0.0, required=False, + validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + value = float(value) + value = Property.validate(self, value) + return value + + def empty(self, value): + return value is None + +class DateTimeProperty(Property): + + data_type = datetime.datetime + type_name = 'DateTime' + + def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, + default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.auto_now = auto_now + self.auto_now_add = auto_now_add + + def default_value(self): + if self.auto_now or self.auto_now_add: + return self.now() + return Property.default_value(self) + + def validate(self, value): + if value == None: + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def get_value_for_datastore(self, model_instance): + if self.auto_now: + setattr(model_instance, self.name, self.now()) + return Property.get_value_for_datastore(self, model_instance) + + def now(self): + return datetime.datetime.utcnow() + +class DateProperty(Property): + + data_type = datetime.date + type_name = 'Date' + + def __init__(self, verbose_name=None, auto_now=False, auto_now_add=False, name=None, + default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.auto_now = auto_now + self.auto_now_add = auto_now_add + + def default_value(self): + if self.auto_now or self.auto_now_add: + return self.now() + return Property.default_value(self) + + def validate(self, value): + if value == None: + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + def get_value_for_datastore(self, model_instance): + if self.auto_now: + setattr(model_instance, self.name, self.now()) + val = Property.get_value_for_datastore(self, model_instance) + if isinstance(val, datetime.datetime): + val = val.date() + return val + + def now(self): + return datetime.date.today() + + +class TimeProperty(Property): + data_type = datetime.time + type_name = 'Time' + + def __init__(self, verbose_name=None, name=None, + default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + + def validate(self, value): + if value is None: + return + if not isinstance(value, self.data_type): + raise TypeError, 'Validation Error, expecting %s, got %s' % (self.data_type, type(value)) + + +class ReferenceProperty(Property): + + data_type = Key + type_name = 'Reference' + + def __init__(self, reference_class=None, collection_name=None, + verbose_name=None, name=None, default=None, required=False, validator=None, choices=None, unique=False): + Property.__init__(self, verbose_name, name, default, required, validator, choices, unique) + self.reference_class = reference_class + self.collection_name = collection_name + + def __get__(self, obj, objtype): + if obj: + value = getattr(obj, self.slot_name) + if value == self.default_value(): + return value + # If the value is still the UUID for the referenced object, we need to create + # the object now that is the attribute has actually been accessed. This lazy + # instantiation saves unnecessary roundtrips to SimpleDB + if isinstance(value, str) or isinstance(value, unicode): + value = self.reference_class(value) + setattr(obj, self.name, value) + return value + + def __set__(self, obj, value): + """Don't allow this object to be associated to itself + This causes bad things to happen""" + if value != None and (obj.id == value or (hasattr(value, "id") and obj.id == value.id)): + raise ValueError, "Can not associate an object with itself!" + return super(ReferenceProperty, self).__set__(obj,value) + + def __property_config__(self, model_class, property_name): + Property.__property_config__(self, model_class, property_name) + if self.collection_name is None: + self.collection_name = '%s_%s_set' % (model_class.__name__.lower(), self.name) + if hasattr(self.reference_class, self.collection_name): + raise ValueError, 'duplicate property: %s' % self.collection_name + setattr(self.reference_class, self.collection_name, + _ReverseReferenceProperty(model_class, property_name, self.collection_name)) + + def check_uuid(self, value): + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + + def check_instance(self, value): + try: + obj_lineage = value.get_lineage() + cls_lineage = self.reference_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not a Model' % value + + def validate(self, value): + if self.required and value==None: + raise ValueError, '%s is a required property' % self.name + if value == self.default_value(): + return + if not isinstance(value, str) and not isinstance(value, unicode): + self.check_instance(value) + +class _ReverseReferenceProperty(Property): + data_type = Query + type_name = 'query' + + def __init__(self, model, prop, name): + self.__model = model + self.__property = prop + self.collection_name = prop + self.name = name + self.item_type = model + + def __get__(self, model_instance, model_class): + """Fetches collection of model instances of this collection property.""" + if model_instance is not None: + query = Query(self.__model) + if type(self.__property) == list: + props = [] + for prop in self.__property: + props.append("%s =" % prop) + return query.filter(props, model_instance) + else: + return query.filter(self.__property + ' =', model_instance) + else: + return self + + def __set__(self, model_instance, value): + """Not possible to set a new collection.""" + raise ValueError, 'Virtual property is read-only' + + +class CalculatedProperty(Property): + + def __init__(self, verbose_name=None, name=None, default=None, + required=False, validator=None, choices=None, + calculated_type=int, unique=False, use_method=False): + Property.__init__(self, verbose_name, name, default, required, + validator, choices, unique) + self.calculated_type = calculated_type + self.use_method = use_method + + def __get__(self, obj, objtype): + value = self.default_value() + if obj: + try: + value = getattr(obj, self.slot_name) + if self.use_method: + value = value() + except AttributeError: + pass + return value + + def __set__(self, obj, value): + """Not possible to set a new AutoID.""" + pass + + def _set_direct(self, obj, value): + if not self.use_method: + setattr(obj, self.slot_name, value) + + def get_value_for_datastore(self, model_instance): + if self.calculated_type in [str, int, bool]: + value = self.__get__(model_instance, model_instance.__class__) + return value + else: + return None + +class ListProperty(Property): + + data_type = list + type_name = 'List' + + def __init__(self, item_type, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = [] + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, list): + value = [value] + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for item in value: + if not isinstance(item, item_type): + if item_type == (int, long): + raise ValueError, 'Items in the %s list must all be integers.' % self.name + else: + raise ValueError('Items in the %s list must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return list(super(ListProperty, self).default_value()) + + def __set__(self, obj, value): + """Override the set method to allow them to set the property to an instance of the item_type instead of requiring a list to be passed in""" + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + if isinstance(value, item_type): + value = [value] + elif value == None: # Override to allow them to set this to "None" to remove everything + value = [] + return super(ListProperty, self).__set__(obj,value) + + +class MapProperty(Property): + + data_type = dict + type_name = 'Map' + + def __init__(self, item_type=str, verbose_name=None, name=None, default=None, **kwds): + if default is None: + default = {} + self.item_type = item_type + Property.__init__(self, verbose_name, name, default=default, required=True, **kwds) + + def validate(self, value): + if value is not None: + if not isinstance(value, dict): + raise ValueError, 'Value must of type dict' + + if self.item_type in (int, long): + item_type = (int, long) + elif self.item_type in (str, unicode): + item_type = (str, unicode) + else: + item_type = self.item_type + + for key in value: + if not isinstance(value[key], item_type): + if item_type == (int, long): + raise ValueError, 'Values in the %s Map must all be integers.' % self.name + else: + raise ValueError('Values in the %s Map must all be %s instances' % + (self.name, self.item_type.__name__)) + return value + + def empty(self, value): + return value is None + + def default_value(self): + return {} diff --git a/backup/src/boto/sdb/db/query.py b/backup/src/boto/sdb/db/query.py new file mode 100644 index 0000000..31b71aa --- /dev/null +++ b/backup/src/boto/sdb/db/query.py @@ -0,0 +1,85 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +class Query(object): + __local_iter__ = None + def __init__(self, model_class, limit=None, next_token=None, manager=None): + self.model_class = model_class + self.limit = limit + self.offset = 0 + if manager: + self.manager = manager + else: + self.manager = self.model_class._manager + self.filters = [] + self.select = None + self.sort_by = None + self.rs = None + self.next_token = next_token + + def __iter__(self): + return iter(self.manager.query(self)) + + def next(self): + if self.__local_iter__ == None: + self.__local_iter__ = self.__iter__() + return self.__local_iter__.next() + + def filter(self, property_operator, value): + self.filters.append((property_operator, value)) + return self + + def fetch(self, limit, offset=0): + """Not currently fully supported, but we can use this + to allow them to set a limit in a chainable method""" + self.limit = limit + self.offset = offset + return self + + def count(self, quick=True): + return self.manager.count(self.model_class, self.filters, quick, self.sort_by, self.select) + + def get_query(self): + return self.manager._build_filter_part(self.model_class, self.filters, self.sort_by, self.select) + + def order(self, key): + self.sort_by = key + return self + + def to_xml(self, doc=None): + if not doc: + xmlmanager = self.model_class.get_xmlmanager() + doc = xmlmanager.new_doc() + for obj in self: + obj.to_xml(doc) + return doc + + def get_next_token(self): + if self.rs: + return self.rs.next_token + if self._next_token: + return self._next_token + return None + + def set_next_token(self, token): + self._next_token = token + + next_token = property(get_next_token, set_next_token) diff --git a/backup/src/boto/sdb/db/sequence.py b/backup/src/boto/sdb/db/sequence.py new file mode 100644 index 0000000..5d7dc10 --- /dev/null +++ b/backup/src/boto/sdb/db/sequence.py @@ -0,0 +1,221 @@ +# Copyright (c) 2010 Chris Moyer http://coredumped.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBResponseError + +class SequenceGenerator(object): + """Generic Sequence Generator object, this takes a single + string as the "sequence" and uses that to figure out + what the next value in a string is. For example + if you give "ABC" and pass in "A" it will give you "B", + and if you give it "C" it will give you "AA". + + If you set "rollover" to True in the above example, passing + in "C" would give you "A" again. + + The Sequence string can be a string or any iterable + that has the "index" function and is indexable. + """ + __name__ = "SequenceGenerator" + + def __init__(self, sequence_string, rollover=False): + """Create a new SequenceGenerator using the sequence_string + as how to generate the next item. + + :param sequence_string: The string or list that explains + how to generate the next item in the sequence + :type sequence_string: str,iterable + + :param rollover: Rollover instead of incrementing when + we hit the end of the sequence + :type rollover: bool + """ + self.sequence_string = sequence_string + self.sequence_length = len(sequence_string[0]) + self.rollover = rollover + self.last_item = sequence_string[-1] + self.__name__ = "%s('%s')" % (self.__class__.__name__, sequence_string) + + def __call__(self, val, last=None): + """Get the next value in the sequence""" + # If they pass us in a string that's not at least + # the lenght of our sequence, then return the + # first element in our sequence + if val == None or len(val) < self.sequence_length: + return self.sequence_string[0] + last_value = val[-self.sequence_length:] + if (not self.rollover) and (last_value == self.last_item): + val = "%s%s" % (self(val[:-self.sequence_length]), self._inc(last_value)) + else: + val = "%s%s" % (val[:-self.sequence_length], self._inc(last_value)) + return val + + def _inc(self, val): + """Increment a single value""" + assert(len(val) == self.sequence_length) + return self.sequence_string[(self.sequence_string.index(val)+1) % len(self.sequence_string)] + + + +# +# Simple Sequence Functions +# +def increment_by_one(cv=None, lv=None): + if cv == None: + return 0 + return cv + 1 + +def double(cv=None, lv=None): + if cv == None: + return 1 + return cv * 2 + +def fib(cv=1, lv=0): + """The fibonacci sequence, this incrementer uses the + last value""" + if cv == None: + cv = 1 + if lv == None: + lv = 0 + return cv + lv + +increment_string = SequenceGenerator("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + + +class Sequence(object): + """A simple Sequence using the new SDB "Consistent" features + Based largly off of the "Counter" example from mitch garnaat: + http://bitbucket.org/mitch/stupidbototricks/src/tip/counter.py""" + + + def __init__(self, id=None, domain_name=None, fnc=increment_by_one, init_val=None): + """Create a new Sequence, using an optional function to + increment to the next number, by default we just increment by one. + Every parameter here is optional, if you don't specify any options + then you'll get a new SequenceGenerator with a random ID stored in the + default domain that increments by one and uses the default botoweb + environment + + :param id: Optional ID (name) for this counter + :type id: str + + :param domain_name: Optional domain name to use, by default we get this out of the + environment configuration + :type domain_name:str + + :param fnc: Optional function to use for the incrementation, by default we just increment by one + There are several functions defined in this module. + Your function must accept "None" to get the initial value + :type fnc: function, str + + :param init_val: Initial value, by default this is the first element in your sequence, + but you can pass in any value, even a string if you pass in a function that uses + strings instead of ints to increment + """ + self._db = None + self._value = None + self.last_value = None + self.domain_name = domain_name + self.id = id + if self.id == None: + import uuid + self.id = str(uuid.uuid4()) + if init_val == None: + init_val = fnc(init_val) + self.val = init_val + + self.item_type = type(fnc(None)) + self.timestamp = None + # Allow us to pass in a full name to a function + if type(fnc) == str: + from boto.utils import find_class + fnc = find_class(fnc) + self.fnc = fnc + + def set(self, val): + """Set the value""" + import time + now = time.time() + expected_value = [] + new_val = {} + new_val['timestamp'] = now + if self._value != None: + new_val['last_value'] = self._value + expected_value = ['current_value', str(self._value)] + new_val['current_value'] = val + try: + self.db.put_attributes(self.id, new_val, expected_value=expected_value) + self.timestamp = new_val['timestamp'] + except SDBResponseError, e: + if e.status == 409: + raise ValueError, "Sequence out of sync" + else: + raise + + + def get(self): + """Get the value""" + val = self.db.get_attributes(self.id, consistent_read=True) + if val and val.has_key('timestamp'): + self.timestamp = val['timestamp'] + if val and val.has_key('current_value'): + self._value = self.item_type(val['current_value']) + if val.has_key("last_value") and val['last_value'] != None: + self.last_value = self.item_type(val['last_value']) + return self._value + + val = property(get, set) + + def __repr__(self): + return "%s('%s', '%s', '%s.%s', '%s')" % ( + self.__class__.__name__, + self.id, + self.domain_name, + self.fnc.__module__, self.fnc.__name__, + self.val) + + + def _connect(self): + """Connect to our domain""" + if not self._db: + if not self.domain_name: + import boto + sdb = boto.connect_sdb() + self.domain_name = boto.config.get("DB", "sequence_db", boto.config.get("DB", "db_name", "default")) + try: + self._db = sdb.get_domain(self.domain_name) + except SDBResponseError, e: + if e.status == 400: + self._db = sdb.create_domain(self.domain_name) + else: + raise + return self._db + + db = property(_connect) + + def next(self): + self.val = self.fnc(self.val, self.last_value) + return self.val + + def delete(self): + """Remove this sequence""" + self.db.delete_attributes(self.id) diff --git a/backup/src/boto/sdb/db/test_db.py b/backup/src/boto/sdb/db/test_db.py new file mode 100644 index 0000000..0c345ab --- /dev/null +++ b/backup/src/boto/sdb/db/test_db.py @@ -0,0 +1,225 @@ +from boto.sdb.db.model import Model +from boto.sdb.db.property import StringProperty, IntegerProperty, BooleanProperty +from boto.sdb.db.property import DateTimeProperty, FloatProperty, ReferenceProperty +from boto.sdb.db.property import PasswordProperty, ListProperty, MapProperty +from datetime import datetime +import time +from boto.exception import SDBPersistenceError + +_objects = {} + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestBasic(Model): + + name = StringProperty() + size = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + +class TestFloat(Model): + + name = StringProperty() + value = FloatProperty() + +class TestRequired(Model): + + req = StringProperty(required=True, default='foo') + +class TestReference(Model): + + ref = ReferenceProperty(reference_class=TestBasic, collection_name='refs') + +class TestSubClass(TestBasic): + + answer = IntegerProperty() + +class TestPassword(Model): + password = PasswordProperty() + +class TestList(Model): + + name = StringProperty() + nums = ListProperty(int) + +class TestMap(Model): + + name = StringProperty() + map = MapProperty() + +class TestListReference(Model): + + name = StringProperty() + basics = ListProperty(TestBasic) + +class TestAutoNow(Model): + + create_date = DateTimeProperty(auto_now_add=True) + modified_date = DateTimeProperty(auto_now=True) + +class TestUnique(Model): + name = StringProperty(unique=True) + +def test_basic(): + global _objects + t = TestBasic() + t.name = 'simple' + t.size = -42 + t.foo = True + t.date = datetime.now() + print 'saving object' + t.put() + _objects['test_basic_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestBasic.get_by_id(t.id) + _objects['test_basic_tt'] = tt + assert tt.id == t.id + l = TestBasic.get_by_id([t.id]) + assert len(l) == 1 + assert l[0].id == t.id + assert t.size == tt.size + assert t.foo == tt.foo + assert t.name == tt.name + #assert t.date == tt.date + return t + +def test_float(): + global _objects + t = TestFloat() + t.name = 'float object' + t.value = 98.6 + print 'saving object' + t.save() + _objects['test_float_t'] = t + time.sleep(5) + print 'now try retrieving it' + tt = TestFloat.get_by_id(t.id) + _objects['test_float_tt'] = tt + assert tt.id == t.id + assert tt.name == t.name + assert tt.value == t.value + return t + +def test_required(): + global _objects + t = TestRequired() + _objects['test_required_t'] = t + t.put() + return t + +def test_reference(t=None): + global _objects + if not t: + t = test_basic() + tt = TestReference() + tt.ref = t + tt.put() + time.sleep(10) + tt = TestReference.get_by_id(tt.id) + _objects['test_reference_tt'] = tt + assert tt.ref.id == t.id + for o in t.refs: + print o + +def test_subclass(): + global _objects + t = TestSubClass() + _objects['test_subclass_t'] = t + t.name = 'a subclass' + t.size = -489 + t.save() + +def test_password(): + global _objects + t = TestPassword() + _objects['test_password_t'] = t + t.password = "foo" + t.save() + time.sleep(5) + # Make sure it stored ok + tt = TestPassword.get_by_id(t.id) + _objects['test_password_tt'] = tt + #Testing password equality + assert tt.password == "foo" + #Testing password not stored as string + assert str(tt.password) != "foo" + +def test_list(): + global _objects + t = TestList() + _objects['test_list_t'] = t + t.name = 'a list of ints' + t.nums = [1,2,3,4,5] + t.put() + tt = TestList.get_by_id(t.id) + _objects['test_list_tt'] = tt + assert tt.name == t.name + for n in tt.nums: + assert isinstance(n, int) + +def test_list_reference(): + global _objects + t = TestBasic() + t.put() + _objects['test_list_ref_t'] = t + tt = TestListReference() + tt.name = "foo" + tt.basics = [t] + tt.put() + time.sleep(5) + _objects['test_list_ref_tt'] = tt + ttt = TestListReference.get_by_id(tt.id) + assert ttt.basics[0].id == t.id + +def test_unique(): + global _objects + t = TestUnique() + name = 'foo' + str(int(time.time())) + t.name = name + t.put() + _objects['test_unique_t'] = t + time.sleep(10) + tt = TestUnique() + _objects['test_unique_tt'] = tt + tt.name = name + try: + tt.put() + assert False + except(SDBPersistenceError): + pass + +def test_datetime(): + global _objects + t = TestAutoNow() + t.put() + _objects['test_datetime_t'] = t + time.sleep(5) + tt = TestAutoNow.get_by_id(t.id) + assert tt.create_date.timetuple() == t.create_date.timetuple() + +def test(): + print 'test_basic' + t1 = test_basic() + print 'test_required' + test_required() + print 'test_reference' + test_reference(t1) + print 'test_subclass' + test_subclass() + print 'test_password' + test_password() + print 'test_list' + test_list() + print 'test_list_reference' + test_list_reference() + print "test_datetime" + test_datetime() + print 'test_unique' + test_unique() + +if __name__ == "__main__": + test() diff --git a/backup/src/boto/sdb/domain.py b/backup/src/boto/sdb/domain.py new file mode 100644 index 0000000..e809124 --- /dev/null +++ b/backup/src/boto/sdb/domain.py @@ -0,0 +1,377 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SDB Domain +""" +from boto.sdb.queryresultset import SelectResultSet + +class Domain: + + def __init__(self, connection=None, name=None): + self.connection = connection + self.name = name + self._metadata = None + + def __repr__(self): + return 'Domain:%s' % self.name + + def __iter__(self): + return iter(self.select("SELECT * FROM `%s`" % self.name)) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'DomainName': + self.name = value + else: + setattr(self, name, value) + + def get_metadata(self): + if not self._metadata: + self._metadata = self.connection.domain_metadata(self) + return self._metadata + + def put_attributes(self, item_name, attributes, + replace=True, expected_value=None): + """ + Store attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being stored. + + :type attribute_names: dict or dict-like object + :param attribute_names: The name/value pairs to store as attributes + + :type expected_value: list + :param expected_value: If supplied, this is a list or tuple consisting + of a single attribute name and expected value. The list can be + of the form: + + * ['name', 'value'] + + In which case the call will first verify that the attribute + "name" of this item has a value of "value". If it does, the delete + will proceed, otherwise a ConditionalCheckFailed error will be + returned. The list can also be of the form: + + * ['name', True|False] + + which will simply check for the existence (True) or non-existence + (False) of the attribute. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.put_attributes(self, item_name, attributes, + replace, expected_value) + + def batch_put_attributes(self, items, replace=True): + """ + Store attributes for multiple items. + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are themselves dictionaries + of attribute names/values, exactly the same as the + attribute_names parameter of the scalar put_attributes + call. + + :type replace: bool + :param replace: Whether the attribute values passed in will replace + existing values or will be added as addition values. + Defaults to True. + + :rtype: bool + :return: True if successful + """ + return self.connection.batch_put_attributes(self, items, replace) + + def get_attributes(self, item_name, attribute_name=None, + consistent_read=False, item=None): + """ + Retrieve attributes for a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being retrieved. + + :type attribute_names: string or list of strings + :param attribute_names: An attribute name or list of attribute names. This + parameter is optional. If not supplied, all attributes + will be retrieved for the item. + + :rtype: :class:`boto.sdb.item.Item` + :return: An Item mapping type containing the requested attribute name/values + """ + return self.connection.get_attributes(self, item_name, attribute_name, + consistent_read, item) + + def delete_attributes(self, item_name, attributes=None, + expected_values=None): + """ + Delete attributes from a given item. + + :type item_name: string + :param item_name: The name of the item whose attributes are being deleted. + + :type attributes: dict, list or :class:`boto.sdb.item.Item` + :param attributes: Either a list containing attribute names which will cause + all values associated with that attribute name to be deleted or + a dict or Item containing the attribute names and keys and list + of values to delete as the value. If no value is supplied, + all attribute name/values for the item will be deleted. + + :type expected_value: list + :param expected_value: If supplied, this is a list or tuple consisting + of a single attribute name and expected value. The list can be of + the form: + + * ['name', 'value'] + + In which case the call will first verify that the attribute "name" + of this item has a value of "value". If it does, the delete + will proceed, otherwise a ConditionalCheckFailed error will be + returned. The list can also be of the form: + + * ['name', True|False] + + which will simply check for the existence (True) or + non-existence (False) of the attribute. + + :rtype: bool + :return: True if successful + """ + return self.connection.delete_attributes(self, item_name, attributes, + expected_values) + + def batch_delete_attributes(self, items): + """ + Delete multiple items in this domain. + + :type items: dict or dict-like object + :param items: A dictionary-like object. The keys of the dictionary are + the item names and the values are either: + + * dictionaries of attribute names/values, exactly the + same as the attribute_names parameter of the scalar + put_attributes call. The attribute name/value pairs + will only be deleted if they match the name/value + pairs passed in. + * None which means that all attributes associated + with the item should be deleted. + + :rtype: bool + :return: True if successful + """ + return self.connection.batch_delete_attributes(self, items) + + def select(self, query='', next_token=None, consistent_read=False, max_items=None): + """ + Returns a set of Attributes for item names within domain_name that match the query. + The query must be expressed in using the SELECT style syntax rather than the + original SimpleDB query language. + + :type query: string + :param query: The SimpleDB query to be performed. + + :rtype: iter + :return: An iterator containing the results. This is actually a generator + function that will iterate across all search results, not just the + first page. + """ + return SelectResultSet(self, query, max_items=max_items, next_token=next_token, + consistent_read=consistent_read) + + def get_item(self, item_name, consistent_read=False): + """ + Retrieves an item from the domain, along with all of its attributes. + + :param string item_name: The name of the item to retrieve. + :rtype: :class:`boto.sdb.item.Item` or ``None`` + :keyword bool consistent_read: When set to true, ensures that the most + recent data is returned. + :return: The requested item, or ``None`` if there was no match found + """ + item = self.get_attributes(item_name, consistent_read=consistent_read) + if item: + item.domain = self + return item + else: + return None + + def new_item(self, item_name): + return self.connection.item_cls(self, item_name) + + def delete_item(self, item): + self.delete_attributes(item.name) + + def to_xml(self, f=None): + """Get this domain as an XML DOM Document + :param f: Optional File to dump directly to + :type f: File or Stream + + :return: File object where the XML has been dumped to + :rtype: file + """ + if not f: + from tempfile import TemporaryFile + f = TemporaryFile() + print >> f, '' + print >> f, '' % self.name + for item in self: + print >> f, '\t' % item.name + for k in item: + print >> f, '\t\t' % k + values = item[k] + if not isinstance(values, list): + values = [values] + for value in values: + print >> f, '\t\t\t> f, ']]>' + print >> f, '\t\t' + print >> f, '\t' + print >> f, '' + f.flush() + f.seek(0) + return f + + + def from_xml(self, doc): + """Load this domain based on an XML document""" + import xml.sax + handler = DomainDumpParser(self) + xml.sax.parse(doc, handler) + return handler + + def delete(self): + """ + Delete this domain, and all items under it + """ + return self.connection.delete(self) + + +class DomainMetaData: + + def __init__(self, domain=None): + self.domain = domain + self.item_count = None + self.item_names_size = None + self.attr_name_count = None + self.attr_names_size = None + self.attr_value_count = None + self.attr_values_size = None + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'ItemCount': + self.item_count = int(value) + elif name == 'ItemNamesSizeBytes': + self.item_names_size = int(value) + elif name == 'AttributeNameCount': + self.attr_name_count = int(value) + elif name == 'AttributeNamesSizeBytes': + self.attr_names_size = int(value) + elif name == 'AttributeValueCount': + self.attr_value_count = int(value) + elif name == 'AttributeValuesSizeBytes': + self.attr_values_size = int(value) + elif name == 'Timestamp': + self.timestamp = value + else: + setattr(self, name, value) + +import sys +from xml.sax.handler import ContentHandler +class DomainDumpParser(ContentHandler): + """ + SAX parser for a domain that has been dumped + """ + + def __init__(self, domain): + self.uploader = UploaderThread(domain) + self.item_id = None + self.attrs = {} + self.attribute = None + self.value = "" + self.domain = domain + + def startElement(self, name, attrs): + if name == "Item": + self.item_id = attrs['id'] + self.attrs = {} + elif name == "attribute": + self.attribute = attrs['id'] + elif name == "value": + self.value = "" + + def characters(self, ch): + self.value += ch + + def endElement(self, name): + if name == "value": + if self.value and self.attribute: + value = self.value.strip() + attr_name = self.attribute.strip() + if self.attrs.has_key(attr_name): + self.attrs[attr_name].append(value) + else: + self.attrs[attr_name] = [value] + elif name == "Item": + self.uploader.items[self.item_id] = self.attrs + # Every 20 items we spawn off the uploader + if len(self.uploader.items) >= 20: + self.uploader.start() + self.uploader = UploaderThread(self.domain) + elif name == "Domain": + # If we're done, spawn off our last Uploader Thread + self.uploader.start() + +from threading import Thread +class UploaderThread(Thread): + """Uploader Thread""" + + def __init__(self, domain): + self.db = domain + self.items = {} + Thread.__init__(self) + + def run(self): + try: + self.db.batch_put_attributes(self.items) + except: + print "Exception using batch put, trying regular put instead" + for item_name in self.items: + self.db.put_attributes(item_name, self.items[item_name]) + print ".", + sys.stdout.flush() diff --git a/backup/src/boto/sdb/item.py b/backup/src/boto/sdb/item.py new file mode 100644 index 0000000..4705b31 --- /dev/null +++ b/backup/src/boto/sdb/item.py @@ -0,0 +1,183 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import base64 + +class Item(dict): + """ + A ``dict`` sub-class that serves as an object representation of a + SimpleDB item. An item in SDB is similar to a row in a relational + database. Items belong to a :py:class:`Domain `, + which is similar to a table in a relational database. + + The keys on instances of this object correspond to attributes that are + stored on the SDB item. + + .. tip:: + While it is possible to instantiate this class directly, you may want + to use the convenience methods on :py:class:`boto.sdb.domain.Domain` + for that purpose. For example, + :py:meth:`boto.sdb.domain.Domain.get_item`. + """ + def __init__(self, domain, name='', active=False): + """ + :type domain: :py:class:`boto.sdb.domain.Domain` + :param domain: The domain that this item belongs to. + + :param str name: The name of this item. This name will be used when + querying for items using methods like + :py:meth:`boto.sdb.domain.Domain.get_item` + """ + dict.__init__(self) + self.domain = domain + self.name = name + self.active = active + self.request_id = None + self.encoding = None + self.in_attribute = False + self.converter = self.domain.connection.converter + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + self.in_attribute = True + self.encoding = attrs.get('encoding', None) + return None + + def decode_value(self, value): + if self.encoding == 'base64': + self.encoding = None + return base64.decodestring(value) + else: + return value + + def endElement(self, name, value, connection): + if name == 'ItemName': + self.name = self.decode_value(value) + elif name == 'Name': + if self.in_attribute: + self.last_key = self.decode_value(value) + else: + self.name = self.decode_value(value) + elif name == 'Value': + if self.has_key(self.last_key): + if not isinstance(self[self.last_key], list): + self[self.last_key] = [self[self.last_key]] + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key].append(value) + else: + value = self.decode_value(value) + if self.converter: + value = self.converter.decode(value) + self[self.last_key] = value + elif name == 'BoxUsage': + try: + connection.box_usage += float(value) + except: + pass + elif name == 'RequestId': + self.request_id = value + elif name == 'Attribute': + self.in_attribute = False + else: + setattr(self, name, value) + + def load(self): + """ + Loads or re-loads this item's attributes from SDB. + + .. warning:: + If you have changed attribute values on an Item instance, + this method will over-write the values if they are different in + SDB. For any local attributes that don't yet exist in SDB, + they will be safe. + """ + self.domain.get_attributes(self.name, item=self) + + def save(self, replace=True): + """ + Saves this item to SDB. + + :param bool replace: If ``True``, delete any attributes on the remote + SDB item that have a ``None`` value on this object. + """ + self.domain.put_attributes(self.name, self, replace) + # Delete any attributes set to "None" + if replace: + del_attrs = [] + for name in self: + if self[name] == None: + del_attrs.append(name) + if len(del_attrs) > 0: + self.domain.delete_attributes(self.name, del_attrs) + + def add_value(self, key, value): + """ + Helps set or add to attributes on this item. If you are adding a new + attribute that has yet to be set, it will simply create an attribute + named ``key`` with your given ``value`` as its value. If you are + adding a value to an existing attribute, this method will convert the + attribute to a list (if it isn't already) and append your new value + to said list. + + For clarification, consider the following interactive session: + + .. code-block:: python + + >>> item = some_domain.get_item('some_item') + >>> item.has_key('some_attr') + False + >>> item.add_value('some_attr', 1) + >>> item['some_attr'] + 1 + >>> item.add_value('some_attr', 2) + >>> item['some_attr'] + [1, 2] + + :param str key: The attribute to add a value to. + :param object value: The value to set or append to the attribute. + """ + if key in self: + # We already have this key on the item. + if not isinstance(self[key], list): + # The key isn't already a list, take its current value and + # convert it to a list with the only member being the + # current value. + self[key] = [self[key]] + # Add the new value to the list. + self[key].append(value) + else: + # This is a new attribute, just set it. + self[key] = value + + def delete(self): + """ + Deletes this item in SDB. + + .. note:: This local Python object remains in its current state + after deletion, this only deletes the remote item in SDB. + """ + self.domain.delete_item(self) + + + + diff --git a/backup/src/boto/sdb/persist/__init__.py b/backup/src/boto/sdb/persist/__init__.py new file mode 100644 index 0000000..2f2b0c1 --- /dev/null +++ b/backup/src/boto/sdb/persist/__init__.py @@ -0,0 +1,83 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.utils import find_class + +class Manager(object): + + DefaultDomainName = boto.config.get('Persist', 'default_domain', None) + + def __init__(self, domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + self.domain_name = domain_name + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + self.domain = None + self.sdb = None + self.s3 = None + if not self.domain_name: + self.domain_name = self.DefaultDomainName + if self.domain_name: + boto.log.info('No SimpleDB domain set, using default_domain: %s' % self.domain_name) + else: + boto.log.warning('No SimpleDB domain set, persistance is disabled') + if self.domain_name: + self.sdb = boto.connect_sdb(aws_access_key_id=self.aws_access_key_id, + aws_secret_access_key=self.aws_secret_access_key, + debug=debug) + self.domain = self.sdb.lookup(self.domain_name) + if not self.domain: + self.domain = self.sdb.create_domain(self.domain_name) + + def get_s3_connection(self): + if not self.s3: + self.s3 = boto.connect_s3(self.aws_access_key_id, self.aws_secret_access_key) + return self.s3 + +def get_manager(domain_name=None, aws_access_key_id=None, aws_secret_access_key=None, debug=0): + return Manager(domain_name, aws_access_key_id, aws_secret_access_key, debug=debug) + +def set_domain(domain_name): + Manager.DefaultDomainName = domain_name + +def get_domain(): + return Manager.DefaultDomainName + +def revive_object_from_id(id, manager): + if not manager.domain: + return None + attrs = manager.domain.get_attributes(id, ['__module__', '__type__', '__lineage__']) + try: + cls = find_class(attrs['__module__'], attrs['__type__']) + return cls(id, manager=manager) + except ImportError: + return None + +def object_lister(cls, query_lister, manager): + for item in query_lister: + if cls: + yield cls(item.name) + else: + o = revive_object_from_id(item.name, manager) + if o: + yield o + + diff --git a/backup/src/boto/sdb/persist/checker.py b/backup/src/boto/sdb/persist/checker.py new file mode 100644 index 0000000..e2146c9 --- /dev/null +++ b/backup/src/boto/sdb/persist/checker.py @@ -0,0 +1,302 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from datetime import datetime +from boto.s3.key import Key +from boto.s3.bucket import Bucket +from boto.sdb.persist import revive_object_from_id +from boto.exception import SDBPersistenceError +from boto.utils import Password + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +class ValueChecker: + + def check(self, value): + """ + Checks a value to see if it is of the right type. + + Should raise a TypeError exception if an in appropriate value is passed in. + """ + raise TypeError + + def from_string(self, str_value, obj): + """ + Takes a string as input and returns the type-specific value represented by that string. + + Should raise a ValueError if the value cannot be converted to the appropriate type. + """ + raise ValueError + + def to_string(self, value): + """ + Convert a value to it's string representation. + + Should raise a ValueError if the value cannot be converted to a string representation. + """ + raise ValueError + +class StringChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.check(params['default']) + self.default = params['default'] + else: + self.default = '' + + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + + def from_string(self, str_value, obj): + return str_value + + def to_string(self, value): + self.check(value) + return value + +class PasswordChecker(StringChecker): + def check(self, value): + if isinstance(value, str) or isinstance(value, unicode) or isinstance(value, Password): + if len(value) > self.maxlength: + raise ValueError, 'Length of value greater than maxlength' + else: + raise TypeError, 'Expecting String, got %s' % type(value) + +class IntegerChecker(ValueChecker): + + __sizes__ = { 'small' : (65535, 32767, -32768, 5), + 'medium' : (4294967295, 2147483647, -2147483648, 10), + 'large' : (18446744073709551615, 9223372036854775807, -9223372036854775808, 20)} + + def __init__(self, **params): + self.size = params.get('size', 'medium') + if self.size not in self.__sizes__.keys(): + raise ValueError, 'size must be one of %s' % self.__sizes__.keys() + self.signed = params.get('signed', True) + self.default = params.get('default', 0) + self.format_string = '%%0%dd' % self.__sizes__[self.size][-1] + + def check(self, value): + if not isinstance(value, int) and not isinstance(value, long): + raise TypeError, 'Expecting int or long, got %s' % type(value) + if self.signed: + min = self.__sizes__[self.size][2] + max = self.__sizes__[self.size][1] + else: + min = 0 + max = self.__sizes__[self.size][0] + if value > max: + raise ValueError, 'Maximum value is %d' % max + if value < min: + raise ValueError, 'Minimum value is %d' % min + + def from_string(self, str_value, obj): + val = int(str_value) + if self.signed: + val = val + self.__sizes__[self.size][2] + return val + + def to_string(self, value): + self.check(value) + if self.signed: + value += -self.__sizes__[self.size][2] + return self.format_string % value + +class BooleanChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('default'): + self.default = params['default'] + else: + self.default = False + + def check(self, value): + if not isinstance(value, bool): + raise TypeError, 'Expecting bool, got %s' % type(value) + + def from_string(self, str_value, obj): + if str_value.lower() == 'true': + return True + else: + return False + + def to_string(self, value): + self.check(value) + if value == True: + return 'true' + else: + return 'false' + +class DateTimeChecker(ValueChecker): + + def __init__(self, **params): + if params.has_key('maxlength'): + self.maxlength = params['maxlength'] + else: + self.maxlength = 1024 + if params.has_key('default'): + self.default = params['default'] + else: + self.default = datetime.now() + + def check(self, value): + if not isinstance(value, datetime): + raise TypeError, 'Expecting datetime, got %s' % type(value) + + def from_string(self, str_value, obj): + try: + return datetime.strptime(str_value, ISO8601) + except: + raise ValueError, 'Unable to convert %s to DateTime' % str_value + + def to_string(self, value): + self.check(value) + return value.strftime(ISO8601) + +class ObjectChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + self.ref_class = params.get('ref_class', None) + if self.ref_class == None: + raise SDBPersistenceError('ref_class parameter is required') + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + # ugly little hack - sometimes I want to just stick a UUID string + # in here rather than instantiate an object. + # This does a bit of hand waving to "type check" the string + t = value.split('-') + if len(t) != 5: + raise ValueError + else: + try: + obj_lineage = value.get_lineage() + cls_lineage = self.ref_class.get_lineage() + if obj_lineage.startswith(cls_lineage): + return + raise TypeError, '%s not instance of %s' % (obj_lineage, cls_lineage) + except: + raise ValueError, '%s is not an SDBObject' % value + + def from_string(self, str_value, obj): + if not str_value: + return None + try: + return revive_object_from_id(str_value, obj._manager) + except: + raise ValueError, 'Unable to convert %s to Object' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return value.id + +class S3KeyChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + try: + bucket_name, key_name = value.split('/', 1) + except: + raise ValueError + elif not isinstance(value, Key): + raise TypeError, 'Expecting Key, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + bucket_name, key_name = str_value.split('/', 1) + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(bucket_name) + key = bucket.get_key(key_name) + if not key: + key = bucket.new_key(key_name) + return key + except: + raise ValueError, 'Unable to convert %s to S3Key' % str_value + + def to_string(self, value): + self.check(value) + if isinstance(value, str) or isinstance(value, unicode): + return value + if value == None: + return '' + else: + return '%s/%s' % (value.bucket.name, value.name) + +class S3BucketChecker(ValueChecker): + + def __init__(self, **params): + self.default = None + + def check(self, value): + if value == None: + return + if isinstance(value, str) or isinstance(value, unicode): + return + elif not isinstance(value, Bucket): + raise TypeError, 'Expecting Bucket, got %s' % type(value) + + def from_string(self, str_value, obj): + if not str_value: + return None + if str_value == 'None': + return None + try: + if obj: + s3 = obj._manager.get_s3_connection() + bucket = s3.get_bucket(str_value) + return bucket + except: + raise ValueError, 'Unable to convert %s to S3Bucket' % str_value + + def to_string(self, value): + self.check(value) + if value == None: + return '' + else: + return '%s' % value.name + diff --git a/backup/src/boto/sdb/persist/object.py b/backup/src/boto/sdb/persist/object.py new file mode 100644 index 0000000..993df1e --- /dev/null +++ b/backup/src/boto/sdb/persist/object.py @@ -0,0 +1,207 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist import get_manager, object_lister +from boto.sdb.persist.property import Property, ScalarProperty +import uuid + +class SDBBase(type): + "Metaclass for all SDBObjects" + def __init__(cls, name, bases, dict): + super(SDBBase, cls).__init__(name, bases, dict) + # Make sure this is a subclass of SDBObject - mainly copied from django ModelBase (thanks!) + try: + if filter(lambda b: issubclass(b, SDBObject), bases): + # look for all of the Properties and set their names + for key in dict.keys(): + if isinstance(dict[key], Property): + property = dict[key] + property.set_name(key) + prop_names = [] + props = cls.properties() + for prop in props: + prop_names.append(prop.name) + setattr(cls, '_prop_names', prop_names) + except NameError: + # 'SDBObject' isn't defined yet, meaning we're looking at our own + # SDBObject class, defined below. + pass + +class SDBObject(object): + __metaclass__ = SDBBase + + _manager = get_manager() + + @classmethod + def get_lineage(cls): + l = [c.__name__ for c in cls.mro()] + l.reverse() + return '.'.join(l) + + @classmethod + def get(cls, id=None, **params): + if params.has_key('manager'): + manager = params['manager'] + else: + manager = cls._manager + if manager.domain and id: + a = cls._manager.domain.get_attributes(id, '__type__') + if a.has_key('__type__'): + return cls(id, manager) + else: + raise SDBPersistenceError('%s object with id=%s does not exist' % (cls.__name__, id)) + else: + rs = cls.find(**params) + try: + obj = rs.next() + except StopIteration: + raise SDBPersistenceError('%s object matching query does not exist' % cls.__name__) + try: + rs.next() + except StopIteration: + return obj + raise SDBPersistenceError('Query matched more than 1 item') + + @classmethod + def find(cls, **params): + if params.has_key('manager'): + manager = params['manager'] + del params['manager'] + else: + manager = cls._manager + keys = params.keys() + if len(keys) > 4: + raise SDBPersistenceError('Too many fields, max is 4') + parts = ["['__type__'='%s'] union ['__lineage__'starts-with'%s']" % (cls.__name__, cls.get_lineage())] + properties = cls.properties() + for key in keys: + found = False + for property in properties: + if property.name == key: + found = True + if isinstance(property, ScalarProperty): + checker = property.checker + parts.append("['%s' = '%s']" % (key, checker.to_string(params[key]))) + else: + raise SDBPersistenceError('%s is not a searchable field' % key) + if not found: + raise SDBPersistenceError('%s is not a valid field' % key) + query = ' intersection '.join(parts) + if manager.domain: + rs = manager.domain.query(query) + else: + rs = [] + return object_lister(None, rs, manager) + + @classmethod + def list(cls, max_items=None, manager=None): + if not manager: + manager = cls._manager + if manager.domain: + rs = manager.domain.query("['__type__' = '%s']" % cls.__name__, max_items=max_items) + else: + rs = [] + return object_lister(cls, rs, manager) + + @classmethod + def properties(cls): + properties = [] + while cls: + for key in cls.__dict__.keys(): + if isinstance(cls.__dict__[key], Property): + properties.append(cls.__dict__[key]) + if len(cls.__bases__) > 0: + cls = cls.__bases__[0] + else: + cls = None + return properties + + # for backwards compatibility + find_properties = properties + + def __init__(self, id=None, manager=None): + if manager: + self._manager = manager + self.id = id + if self.id: + self._auto_update = True + if self._manager.domain: + attrs = self._manager.domain.get_attributes(self.id, '__type__') + if len(attrs.keys()) == 0: + raise SDBPersistenceError('Object %s: not found' % self.id) + else: + self.id = str(uuid.uuid4()) + self._auto_update = False + + def __setattr__(self, name, value): + if name in self._prop_names: + object.__setattr__(self, name, value) + elif name.startswith('_'): + object.__setattr__(self, name, value) + elif name == 'id': + object.__setattr__(self, name, value) + else: + self._persist_attribute(name, value) + object.__setattr__(self, name, value) + + def __getattr__(self, name): + if not name.startswith('_'): + a = self._manager.domain.get_attributes(self.id, name) + if a.has_key(name): + object.__setattr__(self, name, a[name]) + return a[name] + raise AttributeError + + def __repr__(self): + return '%s<%s>' % (self.__class__.__name__, self.id) + + def _persist_attribute(self, name, value): + if self.id: + self._manager.domain.put_attributes(self.id, {name : value}, replace=True) + + def _get_sdb_item(self): + return self._manager.domain.get_item(self.id) + + def save(self): + attrs = {'__type__' : self.__class__.__name__, + '__module__' : self.__class__.__module__, + '__lineage__' : self.get_lineage()} + for property in self.properties(): + attrs[property.name] = property.to_string(self) + if self._manager.domain: + self._manager.domain.put_attributes(self.id, attrs, replace=True) + self._auto_update = True + + def delete(self): + if self._manager.domain: + self._manager.domain.delete_attributes(self.id) + + def get_related_objects(self, ref_name, ref_cls=None): + if self._manager.domain: + query = "['%s' = '%s']" % (ref_name, self.id) + if ref_cls: + query += " intersection ['__type__'='%s']" % ref_cls.__name__ + rs = self._manager.domain.query(query) + else: + rs = [] + return object_lister(ref_cls, rs, self._manager) + diff --git a/backup/src/boto/sdb/persist/property.py b/backup/src/boto/sdb/persist/property.py new file mode 100644 index 0000000..4776d35 --- /dev/null +++ b/backup/src/boto/sdb/persist/property.py @@ -0,0 +1,371 @@ +# Copyright (c) 2006,2007,2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.exception import SDBPersistenceError +from boto.sdb.persist.checker import StringChecker, PasswordChecker, IntegerChecker, BooleanChecker +from boto.sdb.persist.checker import DateTimeChecker, ObjectChecker, S3KeyChecker, S3BucketChecker +from boto.utils import Password + +class Property(object): + + def __init__(self, checker_class, **params): + self.name = '' + self.checker = checker_class(**params) + self.slot_name = '__' + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + +class ScalarProperty(Property): + + def save(self, obj): + domain = obj._manager.domain + domain.put_attributes(obj.id, {self.name : self.to_string(obj)}, replace=True) + + def to_string(self, obj): + return self.checker.to_string(getattr(obj, self.name)) + + def load(self, obj): + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + # try to get the attribute value from SDB + if self.name in a: + value = self.checker.from_string(a[self.name], obj) + setattr(obj, self.slot_name, value) + # if it's not there, set the value to the default value + else: + self.__set__(obj, self.checker.default) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = self.checker.default + setattr(obj, self.slot_name, self.checker.default) + return value + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class StringProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, StringChecker, **params) + +class PasswordProperty(ScalarProperty): + """ + Hashed password + """ + + def __init__(self, **params): + ScalarProperty.__init__(self, PasswordChecker, **params) + + def __set__(self, obj, value): + p = Password() + p.set(value) + ScalarProperty.__set__(self, obj, p) + + def __get__(self, obj, objtype): + return Password(ScalarProperty.__get__(self, obj, objtype)) + +class SmallPositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class SmallIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class IntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + ScalarProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerProperty(ScalarProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + ScalarProperty.__init__(self, IntegerChecker, **params) + +class BooleanProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, BooleanChecker, **params) + +class DateTimeProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, DateTimeChecker, **params) + +class ObjectProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, ObjectChecker, **params) + +class S3KeyProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3KeyChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class S3BucketProperty(ScalarProperty): + + def __init__(self, **params): + ScalarProperty.__init__(self, S3BucketChecker, **params) + + def __set__(self, obj, value): + self.checker.check(value) + try: + old_value = getattr(obj, self.slot_name) + except: + old_value = self.checker.default + if isinstance(value, str): + value = self.checker.from_string(value, obj) + setattr(obj, self.slot_name, value) + if obj._auto_update: + try: + self.save(obj) + except: + setattr(obj, self.slot_name, old_value) + raise + +class MultiValueProperty(Property): + + def __init__(self, checker_class, **params): + Property.__init__(self, checker_class, **params) + + def __get__(self, obj, objtype): + if obj: + try: + value = getattr(obj, self.slot_name) + except AttributeError: + if obj._auto_update: + self.load(obj) + value = getattr(obj, self.slot_name) + else: + value = MultiValue(self, obj, []) + setattr(obj, self.slot_name, value) + return value + + def load(self, obj): + if obj != None: + _list = [] + domain = obj._manager.domain + a = domain.get_attributes(obj.id, self.name) + if self.name in a: + lst = a[self.name] + if not isinstance(lst, list): + lst = [lst] + for value in lst: + value = self.checker.from_string(value, obj) + _list.append(value) + setattr(obj, self.slot_name, MultiValue(self, obj, _list)) + + def __set__(self, obj, value): + if not isinstance(value, list): + raise SDBPersistenceError('Value must be a list') + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + str_list = self.to_string(obj) + domain = obj._manager.domain + if obj._auto_update: + if len(str_list) == 1: + domain.put_attributes(obj.id, {self.name : str_list[0]}, replace=True) + else: + try: + self.__delete__(obj) + except: + pass + domain.put_attributes(obj.id, {self.name : str_list}, replace=True) + setattr(obj, self.slot_name, MultiValue(self, obj, value)) + + def __delete__(self, obj): + if obj._auto_update: + domain = obj._manager.domain + domain.delete_attributes(obj.id, [self.name]) + setattr(obj, self.slot_name, MultiValue(self, obj, [])) + + def to_string(self, obj): + str_list = [] + for value in self.__get__(obj, type(obj)): + str_list.append(self.checker.to_string(value)) + return str_list + +class StringListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, StringChecker, **params) + +class SmallIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class SmallPositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'small' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class IntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class PositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'medium' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargeIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = True + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class LargePositiveIntegerListProperty(MultiValueProperty): + + def __init__(self, **params): + params['size'] = 'large' + params['signed'] = False + MultiValueProperty.__init__(self, IntegerChecker, **params) + +class BooleanListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, BooleanChecker, **params) + +class ObjectListProperty(MultiValueProperty): + + def __init__(self, **params): + MultiValueProperty.__init__(self, ObjectChecker, **params) + +class HasManyProperty(Property): + + def set_name(self, name): + self.name = name + self.slot_name = '__' + self.name + + def __get__(self, obj, objtype): + return self + + +class MultiValue: + """ + Special Multi Value for boto persistence layer to allow us to do + obj.list.append(foo) + """ + def __init__(self, property, obj, _list): + self.checker = property.checker + self.name = property.name + self.object = obj + self._list = _list + + def __repr__(self): + return repr(self._list) + + def __getitem__(self, key): + return self._list.__getitem__(key) + + def __delitem__(self, key): + item = self[key] + self._list.__delitem__(key) + domain = self.object._manager.domain + domain.delete_attributes(self.object.id, {self.name: [self.checker.to_string(item)]}) + + def __len__(self): + return len(self._list) + + def append(self, value): + self.checker.check(value) + self._list.append(value) + domain = self.object._manager.domain + domain.put_attributes(self.object.id, {self.name: self.checker.to_string(value)}, replace=False) + + def index(self, value): + for x in self._list: + if x.id == value.id: + return self._list.index(x) + + def remove(self, value): + del(self[self.index(value)]) diff --git a/backup/src/boto/sdb/persist/test_persist.py b/backup/src/boto/sdb/persist/test_persist.py new file mode 100644 index 0000000..080935d --- /dev/null +++ b/backup/src/boto/sdb/persist/test_persist.py @@ -0,0 +1,141 @@ +from boto.sdb.persist.object import SDBObject +from boto.sdb.persist.property import StringProperty, PositiveIntegerProperty, IntegerProperty +from boto.sdb.persist.property import BooleanProperty, DateTimeProperty, S3KeyProperty +from boto.sdb.persist.property import ObjectProperty, StringListProperty +from boto.sdb.persist.property import PositiveIntegerListProperty, BooleanListProperty, ObjectListProperty +from boto.sdb.persist import Manager +from datetime import datetime +import time + +# +# This will eventually be moved to the boto.tests module and become a real unit test +# but for now it will live here. It shows examples of each of the Property types in +# use and tests the basic operations. +# +class TestScalar(SDBObject): + + name = StringProperty() + description = StringProperty() + size = PositiveIntegerProperty() + offset = IntegerProperty() + foo = BooleanProperty() + date = DateTimeProperty() + file = S3KeyProperty() + +class TestRef(SDBObject): + + name = StringProperty() + ref = ObjectProperty(ref_class=TestScalar) + +class TestSubClass1(TestRef): + + answer = PositiveIntegerProperty() + +class TestSubClass2(TestScalar): + + flag = BooleanProperty() + +class TestList(SDBObject): + + names = StringListProperty() + numbers = PositiveIntegerListProperty() + bools = BooleanListProperty() + objects = ObjectListProperty(ref_class=TestScalar) + +def test1(): + s = TestScalar() + s.name = 'foo' + s.description = 'This is foo' + s.size = 42 + s.offset = -100 + s.foo = True + s.date = datetime.now() + s.save() + return s + +def test2(ref_name): + s = TestRef() + s.name = 'testref' + rs = TestScalar.find(name=ref_name) + s.ref = rs.next() + s.save() + return s + +def test3(): + s = TestScalar() + s.name = 'bar' + s.description = 'This is bar' + s.size = 24 + s.foo = False + s.date = datetime.now() + s.save() + return s + +def test4(ref1, ref2): + s = TestList() + s.names.append(ref1.name) + s.names.append(ref2.name) + s.numbers.append(ref1.size) + s.numbers.append(ref2.size) + s.bools.append(ref1.foo) + s.bools.append(ref2.foo) + s.objects.append(ref1) + s.objects.append(ref2) + s.save() + return s + +def test5(ref): + s = TestSubClass1() + s.answer = 42 + s.ref = ref + s.save() + # test out free form attribute + s.fiddlefaddle = 'this is fiddlefaddle' + s._fiddlefaddle = 'this is not fiddlefaddle' + return s + +def test6(): + s = TestSubClass2() + s.name = 'fie' + s.description = 'This is fie' + s.size = 4200 + s.offset = -820 + s.foo = False + s.date = datetime.now() + s.flag = True + s.save() + return s + +def test(domain_name): + print 'Initialize the Persistance system' + Manager.DefaultDomainName = domain_name + print 'Call test1' + s1 = test1() + # now create a new instance and read the saved data from SDB + print 'Now sleep to wait for things to converge' + time.sleep(5) + print 'Now lookup the object and compare the fields' + s2 = TestScalar(s1.id) + assert s1.name == s2.name + assert s1.description == s2.description + assert s1.size == s2.size + assert s1.offset == s2.offset + assert s1.foo == s2.foo + #assert s1.date == s2.date + print 'Call test2' + s2 = test2(s1.name) + print 'Call test3' + s3 = test3() + print 'Call test4' + s4 = test4(s1, s3) + print 'Call test5' + s6 = test6() + s5 = test5(s6) + domain = s5._manager.domain + item1 = domain.get_item(s1.id) + item2 = domain.get_item(s2.id) + item3 = domain.get_item(s3.id) + item4 = domain.get_item(s4.id) + item5 = domain.get_item(s5.id) + item6 = domain.get_item(s6.id) + return [(s1, item1), (s2, item2), (s3, item3), (s4, item4), (s5, item5), (s6, item6)] diff --git a/backup/src/boto/sdb/queryresultset.py b/backup/src/boto/sdb/queryresultset.py new file mode 100644 index 0000000..10bafd1 --- /dev/null +++ b/backup/src/boto/sdb/queryresultset.py @@ -0,0 +1,92 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +def query_lister(domain, query='', max_items=None, attr_names=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.query_with_attributes(domain, query, attr_names, + next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class QueryResultSet: + + def __init__(self, domain=None, query='', max_items=None, attr_names=None): + self.max_items = max_items + self.domain = domain + self.query = query + self.attr_names = attr_names + + def __iter__(self): + return query_lister(self.domain, self.query, self.max_items, self.attr_names) + +def select_lister(domain, query='', max_items=None): + more_results = True + num_results = 0 + next_token = None + while more_results: + rs = domain.connection.select(domain, query, next_token=next_token) + for item in rs: + if max_items: + if num_results == max_items: + raise StopIteration + yield item + num_results += 1 + next_token = rs.next_token + more_results = next_token != None + +class SelectResultSet(object): + + def __init__(self, domain=None, query='', max_items=None, + next_token=None, consistent_read=False): + self.domain = domain + self.query = query + self.consistent_read = consistent_read + self.max_items = max_items + self.next_token = next_token + + def __iter__(self): + more_results = True + num_results = 0 + while more_results: + rs = self.domain.connection.select(self.domain, self.query, + next_token=self.next_token, + consistent_read=self.consistent_read) + for item in rs: + if self.max_items and num_results >= self.max_items: + raise StopIteration + yield item + num_results += 1 + self.next_token = rs.next_token + if self.max_items and num_results >= self.max_items: + raise StopIteration + more_results = self.next_token != None + + def next(self): + return self.__iter__().next() diff --git a/backup/src/boto/sdb/regioninfo.py b/backup/src/boto/sdb/regioninfo.py new file mode 100644 index 0000000..5c32864 --- /dev/null +++ b/backup/src/boto/sdb/regioninfo.py @@ -0,0 +1,32 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.regioninfo import RegionInfo + +class SDBRegionInfo(RegionInfo): + + def __init__(self, connection=None, name=None, endpoint=None): + from boto.sdb.connection import SDBConnection + RegionInfo.__init__(self, connection, name, endpoint, + SDBConnection) diff --git a/backup/src/boto/services/__init__.py b/backup/src/boto/services/__init__.py new file mode 100644 index 0000000..449bd16 --- /dev/null +++ b/backup/src/boto/services/__init__.py @@ -0,0 +1,23 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + + diff --git a/backup/src/boto/services/bs.py b/backup/src/boto/services/bs.py new file mode 100644 index 0000000..3d70031 --- /dev/null +++ b/backup/src/boto/services/bs.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +from optparse import OptionParser +from boto.services.servicedef import ServiceDef +from boto.services.submit import Submitter +from boto.services.result import ResultProcessor +import boto +import sys, os, StringIO + +class BS(object): + + Usage = "usage: %prog [options] config_file command" + + Commands = {'reset' : 'Clear input queue and output bucket', + 'submit' : 'Submit local files to the service', + 'start' : 'Start the service', + 'status' : 'Report on the status of the service buckets and queues', + 'retrieve' : 'Retrieve output generated by a batch', + 'batches' : 'List all batches stored in current output_domain'} + + def __init__(self): + self.service_name = None + self.parser = OptionParser(usage=self.Usage) + self.parser.add_option("--help-commands", action="store_true", dest="help_commands", + help="provides help on the available commands") + self.parser.add_option("-a", "--access-key", action="store", type="string", + help="your AWS Access Key") + self.parser.add_option("-s", "--secret-key", action="store", type="string", + help="your AWS Secret Access Key") + self.parser.add_option("-p", "--path", action="store", type="string", dest="path", + help="the path to local directory for submit and retrieve") + self.parser.add_option("-k", "--keypair", action="store", type="string", dest="keypair", + help="the SSH keypair used with launched instance(s)") + self.parser.add_option("-l", "--leave", action="store_true", dest="leave", + help="leave the files (don't retrieve) files during retrieve command") + self.parser.set_defaults(leave=False) + self.parser.add_option("-n", "--num-instances", action="store", type="string", dest="num_instances", + help="the number of launched instance(s)") + self.parser.set_defaults(num_instances=1) + self.parser.add_option("-i", "--ignore-dirs", action="append", type="string", dest="ignore", + help="directories that should be ignored by submit command") + self.parser.add_option("-b", "--batch-id", action="store", type="string", dest="batch", + help="batch identifier required by the retrieve command") + + def print_command_help(self): + print '\nCommands:' + for key in self.Commands.keys(): + print ' %s\t\t%s' % (key, self.Commands[key]) + + def do_reset(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'clearing out input queue' + i = 0 + m = iq.read() + while m: + i += 1 + iq.delete_message(m) + m = iq.read() + print 'deleted %d messages' % i + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + print 'delete generated files in output bucket' + i = 0 + for k in ob: + i += 1 + k.delete() + print 'deleted %d keys' % i + + def do_submit(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + s = Submitter(self.sd) + t = s.submit_path(self.options.path, None, self.options.ignore, None, + None, True, self.options.path) + print 'A total of %d files were submitted' % t[1] + print 'Batch Identifier: %s' % t[0] + + def do_start(self): + ami_id = self.sd.get('ami_id') + instance_type = self.sd.get('instance_type', 'm1.small') + security_group = self.sd.get('security_group', 'default') + if not ami_id: + self.parser.error('ami_id option is required when starting the service') + ec2 = boto.connect_ec2() + if not self.sd.has_section('Credentials'): + self.sd.add_section('Credentials') + self.sd.set('Credentials', 'aws_access_key_id', ec2.aws_access_key_id) + self.sd.set('Credentials', 'aws_secret_access_key', ec2.aws_secret_access_key) + s = StringIO.StringIO() + self.sd.write(s) + rs = ec2.get_all_images([ami_id]) + img = rs[0] + r = img.run(user_data=s.getvalue(), key_name=self.options.keypair, + max_count=self.options.num_instances, + instance_type=instance_type, + security_groups=[security_group]) + print 'Starting AMI: %s' % ami_id + print 'Reservation %s contains the following instances:' % r.id + for i in r.instances: + print '\t%s' % i.id + + def do_status(self): + iq = self.sd.get_obj('input_queue') + if iq: + print 'The input_queue (%s) contains approximately %s messages' % (iq.id, iq.count()) + ob = self.sd.get_obj('output_bucket') + ib = self.sd.get_obj('input_bucket') + if ob: + if ib and ob.name == ib.name: + return + total = 0 + for k in ob: + total += 1 + print 'The output_bucket (%s) contains %d keys' % (ob.name, total) + + def do_retrieve(self): + if not self.options.path: + self.parser.error('No path provided') + if not os.path.exists(self.options.path): + self.parser.error('Invalid path (%s)' % self.options.path) + if not self.options.batch: + self.parser.error('batch identifier is required for retrieve command') + s = ResultProcessor(self.options.batch, self.sd) + s.get_results(self.options.path, get_file=(not self.options.leave)) + + def do_batches(self): + d = self.sd.get_obj('output_domain') + if d: + print 'Available Batches:' + rs = d.query("['type'='Batch']") + for item in rs: + print ' %s' % item.name + else: + self.parser.error('No output_domain specified for service') + + def main(self): + self.options, self.args = self.parser.parse_args() + if self.options.help_commands: + self.print_command_help() + sys.exit(0) + if len(self.args) != 2: + self.parser.error("config_file and command are required") + self.config_file = self.args[0] + self.sd = ServiceDef(self.config_file) + self.command = self.args[1] + if hasattr(self, 'do_%s' % self.command): + method = getattr(self, 'do_%s' % self.command) + method() + else: + self.parser.error('command (%s) not recognized' % self.command) + +if __name__ == "__main__": + bs = BS() + bs.main() diff --git a/backup/src/boto/services/message.py b/backup/src/boto/services/message.py new file mode 100644 index 0000000..79f6d19 --- /dev/null +++ b/backup/src/boto/services/message.py @@ -0,0 +1,58 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sqs.message import MHMessage +from boto.utils import get_ts +from socket import gethostname +import os, mimetypes, time + +class ServiceMessage(MHMessage): + + def for_key(self, key, params=None, bucket_name=None): + if params: + self.update(params) + if key.path: + t = os.path.split(key.path) + self['OriginalLocation'] = t[0] + self['OriginalFileName'] = t[1] + mime_type = mimetypes.guess_type(t[1])[0] + if mime_type == None: + mime_type = 'application/octet-stream' + self['Content-Type'] = mime_type + s = os.stat(key.path) + t = time.gmtime(s[7]) + self['FileAccessedDate'] = get_ts(t) + t = time.gmtime(s[8]) + self['FileModifiedDate'] = get_ts(t) + t = time.gmtime(s[9]) + self['FileCreateDate'] = get_ts(t) + else: + self['OriginalFileName'] = key.name + self['OriginalLocation'] = key.bucket.name + self['ContentType'] = key.content_type + self['Host'] = gethostname() + if bucket_name: + self['Bucket'] = bucket_name + else: + self['Bucket'] = key.bucket.name + self['InputKey'] = key.name + self['Size'] = key.size + diff --git a/backup/src/boto/services/result.py b/backup/src/boto/services/result.py new file mode 100644 index 0000000..32a6d6a --- /dev/null +++ b/backup/src/boto/services/result.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os +from datetime import datetime, timedelta +from boto.utils import parse_ts +import boto + +class ResultProcessor: + + LogFileName = 'log.csv' + + def __init__(self, batch_name, sd, mimetype_files=None): + self.sd = sd + self.batch = batch_name + self.log_fp = None + self.num_files = 0 + self.total_time = 0 + self.min_time = timedelta.max + self.max_time = timedelta.min + self.earliest_time = datetime.max + self.latest_time = datetime.min + self.queue = self.sd.get_obj('output_queue') + self.domain = self.sd.get_obj('output_domain') + + def calculate_stats(self, msg): + start_time = parse_ts(msg['Service-Read']) + end_time = parse_ts(msg['Service-Write']) + elapsed_time = end_time - start_time + if elapsed_time > self.max_time: + self.max_time = elapsed_time + if elapsed_time < self.min_time: + self.min_time = elapsed_time + self.total_time += elapsed_time.seconds + if start_time < self.earliest_time: + self.earliest_time = start_time + if end_time > self.latest_time: + self.latest_time = end_time + + def log_message(self, msg, path): + keys = msg.keys() + keys.sort() + if not self.log_fp: + self.log_fp = open(os.path.join(path, self.LogFileName), 'a') + line = ','.join(keys) + self.log_fp.write(line+'\n') + values = [] + for key in keys: + value = msg[key] + if value.find(',') > 0: + value = '"%s"' % value + values.append(value) + line = ','.join(values) + self.log_fp.write(line+'\n') + + def process_record(self, record, path, get_file=True): + self.log_message(record, path) + self.calculate_stats(record) + outputs = record['OutputKey'].split(',') + if record.has_key('OutputBucket'): + bucket = boto.lookup('s3', record['OutputBucket']) + else: + bucket = boto.lookup('s3', record['Bucket']) + for output in outputs: + if get_file: + key_name = output.split(';')[0] + key = bucket.lookup(key_name) + file_name = os.path.join(path, key_name) + print 'retrieving file: %s to %s' % (key_name, file_name) + key.get_contents_to_filename(file_name) + self.num_files += 1 + + def get_results_from_queue(self, path, get_file=True, delete_msg=True): + m = self.queue.read() + while m: + if m.has_key('Batch') and m['Batch'] == self.batch: + self.process_record(m, path, get_file) + if delete_msg: + self.queue.delete_message(m) + m = self.queue.read() + + def get_results_from_domain(self, path, get_file=True): + rs = self.domain.query("['Batch'='%s']" % self.batch) + for item in rs: + self.process_record(item, path, get_file) + + def get_results_from_bucket(self, path): + bucket = self.sd.get_obj('output_bucket') + if bucket: + print 'No output queue or domain, just retrieving files from output_bucket' + for key in bucket: + file_name = os.path.join(path, key) + print 'retrieving file: %s to %s' % (key, file_name) + key.get_contents_to_filename(file_name) + self.num_files + 1 + + def get_results(self, path, get_file=True, delete_msg=True): + if not os.path.isdir(path): + os.mkdir(path) + if self.queue: + self.get_results_from_queue(path, get_file) + elif self.domain: + self.get_results_from_domain(path, get_file) + else: + self.get_results_from_bucket(path) + if self.log_fp: + self.log_fp.close() + print '%d results successfully retrieved.' % self.num_files + if self.num_files > 0: + self.avg_time = float(self.total_time)/self.num_files + print 'Minimum Processing Time: %d' % self.min_time.seconds + print 'Maximum Processing Time: %d' % self.max_time.seconds + print 'Average Processing Time: %f' % self.avg_time + self.elapsed_time = self.latest_time-self.earliest_time + print 'Elapsed Time: %d' % self.elapsed_time.seconds + tput = 1.0 / ((self.elapsed_time.seconds/60.0) / self.num_files) + print 'Throughput: %f transactions / minute' % tput + diff --git a/backup/src/boto/services/service.py b/backup/src/boto/services/service.py new file mode 100644 index 0000000..8ee1a8b --- /dev/null +++ b/backup/src/boto/services/service.py @@ -0,0 +1,161 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.message import ServiceMessage +from boto.services.servicedef import ServiceDef +from boto.pyami.scriptbase import ScriptBase +from boto.utils import get_ts +import time +import os +import mimetypes + + +class Service(ScriptBase): + + # Time required to process a transaction + ProcessingTime = 60 + + def __init__(self, config_file=None, mimetype_files=None): + ScriptBase.__init__(self, config_file) + self.name = self.__class__.__name__ + self.working_dir = boto.config.get('Pyami', 'working_dir') + self.sd = ServiceDef(config_file) + self.retry_count = self.sd.getint('retry_count', 5) + self.loop_delay = self.sd.getint('loop_delay', 30) + self.processing_time = self.sd.getint('processing_time', 60) + self.input_queue = self.sd.get_obj('input_queue') + self.output_queue = self.sd.get_obj('output_queue') + self.output_domain = self.sd.get_obj('output_domain') + if mimetype_files: + mimetypes.init(mimetype_files) + + def split_key(key): + if key.find(';') < 0: + t = (key, '') + else: + key, type = key.split(';') + label, mtype = type.split('=') + t = (key, mtype) + return t + + def read_message(self): + boto.log.info('read_message') + message = self.input_queue.read(self.processing_time) + if message: + boto.log.info(message.get_body()) + key = 'Service-Read' + message[key] = get_ts() + return message + + # retrieve the source file from S3 + def get_file(self, message): + bucket_name = message['Bucket'] + key_name = message['InputKey'] + file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file')) + boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.get_contents_to_filename(os.path.join(self.working_dir, file_name)) + return file_name + + # process source file, return list of output files + def process_file(self, in_file_name, msg): + return [] + + # store result file in S3 + def put_file(self, bucket_name, file_path, key_name=None): + boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name)) + bucket = boto.lookup('s3', bucket_name) + key = bucket.new_key(key_name) + key.set_contents_from_filename(file_path) + return key + + def save_results(self, results, input_message, output_message): + output_keys = [] + for file, type in results: + if input_message.has_key('OutputBucket'): + output_bucket = input_message['OutputBucket'] + else: + output_bucket = input_message['Bucket'] + key_name = os.path.split(file)[1] + key = self.put_file(output_bucket, file, key_name) + output_keys.append('%s;type=%s' % (key.name, type)) + output_message['OutputKey'] = ','.join(output_keys) + + # write message to each output queue + def write_message(self, message): + message['Service-Write'] = get_ts() + message['Server'] = self.name + if os.environ.has_key('HOSTNAME'): + message['Host'] = os.environ['HOSTNAME'] + else: + message['Host'] = 'unknown' + message['Instance-ID'] = self.instance_id + if self.output_queue: + boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id) + self.output_queue.write(message) + if self.output_domain: + boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name) + item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']]) + self.output_domain.put_attributes(item_name, message) + + # delete message from input queue + def delete_message(self, message): + boto.log.info('deleting message from %s' % self.input_queue.id) + self.input_queue.delete_message(message) + + # to clean up any files, etc. after each iteration + def cleanup(self): + pass + + def shutdown(self): + on_completion = self.sd.get('on_completion', 'shutdown') + if on_completion == 'shutdown': + if self.instance_id: + time.sleep(60) + c = boto.connect_ec2() + c.terminate_instances([self.instance_id]) + + def main(self, notify=False): + self.notify('Service: %s Starting' % self.name) + empty_reads = 0 + while self.retry_count < 0 or empty_reads < self.retry_count: + try: + input_message = self.read_message() + if input_message: + empty_reads = 0 + output_message = ServiceMessage(None, input_message.get_body()) + input_file = self.get_file(input_message) + results = self.process_file(input_file, output_message) + self.save_results(results, input_message, output_message) + self.write_message(output_message) + self.delete_message(input_message) + self.cleanup() + else: + empty_reads += 1 + time.sleep(self.loop_delay) + except Exception: + boto.log.exception('Service Failed') + empty_reads += 1 + self.notify('Service: %s Shutting Down' % self.name) + self.shutdown() + diff --git a/backup/src/boto/services/servicedef.py b/backup/src/boto/services/servicedef.py new file mode 100644 index 0000000..1cb01aa --- /dev/null +++ b/backup/src/boto/services/servicedef.py @@ -0,0 +1,91 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.pyami.config import Config +from boto.services.message import ServiceMessage +import boto + +class ServiceDef(Config): + + def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None): + Config.__init__(self, config_file) + self.aws_access_key_id = aws_access_key_id + self.aws_secret_access_key = aws_secret_access_key + script = Config.get(self, 'Pyami', 'scripts') + if script: + self.name = script.split('.')[-1] + else: + self.name = None + + + def get(self, name, default=None): + return Config.get(self, self.name, name, default) + + def has_option(self, option): + return Config.has_option(self, self.name, option) + + def getint(self, option, default=0): + try: + val = Config.get(self, self.name, option) + val = int(val) + except: + val = int(default) + return val + + def getbool(self, option, default=False): + try: + val = Config.get(self, self.name, option) + if val.lower() == 'true': + val = True + else: + val = False + except: + val = default + return val + + def get_obj(self, name): + """ + Returns the AWS object associated with a given option. + + The heuristics used are a bit lame. If the option name contains + the word 'bucket' it is assumed to be an S3 bucket, if the name + contains the word 'queue' it is assumed to be an SQS queue and + if it contains the word 'domain' it is assumed to be a SimpleDB + domain. If the option name specified does not exist in the + config file or if the AWS object cannot be retrieved this + returns None. + """ + val = self.get(name) + if not val: + return None + if name.find('queue') >= 0: + obj = boto.lookup('sqs', val) + if obj: + obj.set_message_class(ServiceMessage) + elif name.find('bucket') >= 0: + obj = boto.lookup('s3', val) + elif name.find('domain') >= 0: + obj = boto.lookup('sdb', val) + else: + obj = None + return obj + + diff --git a/backup/src/boto/services/sonofmmm.cfg b/backup/src/boto/services/sonofmmm.cfg new file mode 100644 index 0000000..d70d379 --- /dev/null +++ b/backup/src/boto/services/sonofmmm.cfg @@ -0,0 +1,43 @@ +# +# Your AWS Credentials +# You only need to supply these in this file if you are not using +# the boto tools to start your service +# +#[Credentials] +#aws_access_key_id = +#aws_secret_access_key = + +# +# Fill out this section if you want emails from the service +# when it starts and stops +# +#[Notification] +#smtp_host = +#smtp_user = +#smtp_pass = +#smtp_from = +#smtp_to = + +[Pyami] +scripts = boto.services.sonofmmm.SonOfMMM + +[SonOfMMM] +# id of the AMI to be launched +ami_id = ami-dc799cb5 +# number of times service will read an empty queue before exiting +# a negative value will cause the service to run forever +retry_count = 5 +# seconds to wait after empty queue read before reading again +loop_delay = 10 +# average time it takes to process a transaction +# controls invisibility timeout of messages +processing_time = 60 +ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s +output_mimetype = video/quicktime +output_ext = .mov +input_bucket = +output_bucket = +output_domain = +output_queue = +input_queue = + diff --git a/backup/src/boto/services/sonofmmm.py b/backup/src/boto/services/sonofmmm.py new file mode 100644 index 0000000..acb7e61 --- /dev/null +++ b/backup/src/boto/services/sonofmmm.py @@ -0,0 +1,81 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import boto +from boto.services.service import Service +from boto.services.message import ServiceMessage +import os +import mimetypes + +class SonOfMMM(Service): + + def __init__(self, config_file=None): + Service.__init__(self, config_file) + self.log_file = '%s.log' % self.instance_id + self.log_path = os.path.join(self.working_dir, self.log_file) + boto.set_file_logger(self.name, self.log_path) + if self.sd.has_option('ffmpeg_args'): + self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args') + else: + self.command = '/usr/local/bin/ffmpeg -y -i %s %s' + self.output_mimetype = self.sd.get('output_mimetype') + if self.sd.has_option('output_ext'): + self.output_ext = self.sd.get('output_ext') + else: + self.output_ext = mimetypes.guess_extension(self.output_mimetype) + self.output_bucket = self.sd.get_obj('output_bucket') + self.input_bucket = self.sd.get_obj('input_bucket') + # check to see if there are any messages queue + # if not, create messages for all files in input_bucket + m = self.input_queue.read(1) + if not m: + self.queue_files() + + def queue_files(self): + boto.log.info('Queueing files from %s' % self.input_bucket.name) + for key in self.input_bucket: + boto.log.info('Queueing %s' % key.name) + m = ServiceMessage() + if self.output_bucket: + d = {'OutputBucket' : self.output_bucket.name} + else: + d = None + m.for_key(key, d) + self.input_queue.write(m) + + def process_file(self, in_file_name, msg): + base, ext = os.path.splitext(in_file_name) + out_file_name = os.path.join(self.working_dir, + base+self.output_ext) + command = self.command % (in_file_name, out_file_name) + boto.log.info('running:\n%s' % command) + status = self.run(command) + if status == 0: + return [(out_file_name, self.output_mimetype)] + else: + return [] + + def shutdown(self): + if os.path.isfile(self.log_path): + if self.output_bucket: + key = self.output_bucket.new_key(self.log_file) + key.set_contents_from_filename(self.log_path) + Service.shutdown(self) diff --git a/backup/src/boto/services/submit.py b/backup/src/boto/services/submit.py new file mode 100644 index 0000000..89c439c --- /dev/null +++ b/backup/src/boto/services/submit.py @@ -0,0 +1,88 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import time +import os + + +class Submitter: + + def __init__(self, sd): + self.sd = sd + self.input_bucket = self.sd.get_obj('input_bucket') + self.output_bucket = self.sd.get_obj('output_bucket') + self.output_domain = self.sd.get_obj('output_domain') + self.queue = self.sd.get_obj('input_queue') + + def get_key_name(self, fullpath, prefix): + key_name = fullpath[len(prefix):] + l = key_name.split(os.sep) + return '/'.join(l) + + def write_message(self, key, metadata): + if self.queue: + m = self.queue.new_message() + m.for_key(key, metadata) + if self.output_bucket: + m['OutputBucket'] = self.output_bucket.name + self.queue.write(m) + + def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'): + if not metadata: + metadata = {} + key_name = self.get_key_name(path, prefix) + k = self.input_bucket.new_key(key_name) + k.update_metadata(metadata) + k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb) + self.write_message(k, metadata) + + def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'): + path = os.path.expanduser(path) + path = os.path.expandvars(path) + path = os.path.abspath(path) + total = 0 + metadata = {} + if tags: + metadata['Tags'] = tags + l = [] + for t in time.gmtime(): + l.append(str(t)) + metadata['Batch'] = '_'.join(l) + if self.output_domain: + self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'}) + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + if ignore_dirs: + for ignore in ignore_dirs: + if ignore in dirs: + dirs.remove(ignore) + for file in files: + fullpath = os.path.join(root, file) + if status: + print 'Submitting %s' % fullpath + self.submit_file(fullpath, metadata, cb, num_cb, prefix) + total += 1 + elif os.path.isfile(path): + self.submit_file(path, metadata, cb, num_cb) + total += 1 + else: + print 'problem with %s' % path + return (metadata['Batch'], total) diff --git a/backup/src/boto/ses/__init__.py b/backup/src/boto/ses/__init__.py new file mode 100644 index 0000000..167080b --- /dev/null +++ b/backup/src/boto/ses/__init__.py @@ -0,0 +1,24 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2011 Harry Marr http://hmarr.com/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from connection import SESConnection + diff --git a/backup/src/boto/ses/connection.py b/backup/src/boto/ses/connection.py new file mode 100644 index 0000000..8268e4a --- /dev/null +++ b/backup/src/boto/ses/connection.py @@ -0,0 +1,270 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2011 Harry Marr http://hmarr.com/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSAuthConnection +from boto.exception import BotoServerError +import boto +import boto.jsonresponse + +import urllib +import base64 + + +class SESConnection(AWSAuthConnection): + + ResponseError = BotoServerError + DefaultHost = 'email.us-east-1.amazonaws.com' + APIVersion = '2010-12-01' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + port=None, proxy=None, proxy_port=None, + host=DefaultHost, debug=0): + AWSAuthConnection.__init__(self, host, aws_access_key_id, + aws_secret_access_key, True, port, proxy, + proxy_port, debug=debug) + + def _required_auth_capability(self): + return ['ses'] + + def _build_list_params(self, params, items, label): + """Add an AWS API-compatible parameter list to a dictionary. + + :type params: dict + :param params: The parameter dictionary + + :type items: list + :param items: Items to be included in the list + + :type label: string + :param label: The parameter list's name + """ + if isinstance(items, str): + items = [items] + for i in range(1, len(items) + 1): + params['%s.%d' % (label, i)] = items[i - 1] + + + def _make_request(self, action, params=None): + """Make a call to the SES API. + + :type action: string + :param action: The API method to use (e.g. SendRawEmail) + + :type params: dict + :param params: Parameters that will be sent as POST data with the API + call. + """ + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + params = params or {} + params['Action'] = action + response = super(SESConnection, self).make_request( + 'POST', + '/', + headers=headers, + data=urllib.urlencode(params) + ) + body = response.read() + if response.status == 200: + list_markers = ('VerifiedEmailAddresses', 'SendDataPoints') + e = boto.jsonresponse.Element(list_marker=list_markers) + h = boto.jsonresponse.XmlHandler(e, None) + h.parse(body) + return e + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + + def send_email(self, source, subject, body, to_addresses, cc_addresses=None, + bcc_addresses=None, format='text', reply_addresses=None, + return_path=None): + """Composes an email message based on input data, and then immediately + queues the message for sending. + + :type source: string + :param source: The sender's email address. + + :type subject: string + :param subject: The subject of the message: A short summary of the + content, which will appear in the recipient's inbox. + + :type body: string + :param body: The message body. + + :type to_addresses: list of strings or string + :param to_addresses: The To: field(s) of the message. + + :type cc_addresses: list of strings or string + :param cc_addresses: The CC: field(s) of the message. + + :type bcc_addresses: list of strings or string + :param bcc_addresses: The BCC: field(s) of the message. + + :type format: string + :param format: The format of the message's body, must be either "text" + or "html". + + :type reply_addresses: list of strings or string + :param reply_addresses: The reply-to email address(es) for the + message. If the recipient replies to the + message, each reply-to address will + receive the reply. + + :type return_path: string + :param return_path: The email address to which bounce notifications are + to be forwarded. If the message cannot be delivered + to the recipient, then an error message will be + returned from the recipient's ISP; this message will + then be forwarded to the email address specified by + the ReturnPath parameter. + + """ + params = { + 'Source': source, + 'Message.Subject.Data': subject, + } + + if return_path: + params['ReturnPath'] = return_path + + format = format.lower().strip() + if format == 'html': + params['Message.Body.Html.Data'] = body + elif format == 'text': + params['Message.Body.Text.Data'] = body + else: + raise ValueError("'format' argument must be 'text' or 'html'") + + self._build_list_params(params, to_addresses, + 'Destination.ToAddresses.member') + if cc_addresses: + self._build_list_params(params, cc_addresses, + 'Destination.CcAddresses.member') + + if bcc_addresses: + self._build_list_params(params, bcc_addresses, + 'Destination.BccAddresses.member') + + if reply_addresses: + self._build_list_params(params, reply_addresses, + 'ReplyToAddresses.member') + + return self._make_request('SendEmail', params) + + def send_raw_email(self, source, raw_message, destinations=None): + """Sends an email message, with header and content specified by the + client. The SendRawEmail action is useful for sending multipart MIME + emails, with attachments or inline content. The raw text of the message + must comply with Internet email standards; otherwise, the message + cannot be sent. + + :type source: string + :param source: The sender's email address. + + :type raw_message: string + :param raw_message: The raw text of the message. The client is + responsible for ensuring the following: + + - Message must contain a header and a body, separated by a blank line. + - All required header fields must be present. + - Each part of a multipart MIME message must be formatted properly. + - MIME content types must be among those supported by Amazon SES. + Refer to the Amazon SES Developer Guide for more details. + - Content must be base64-encoded, if MIME requires it. + + :type destinations: list of strings or string + :param destinations: A list of destinations for the message. + + """ + params = { + 'Source': source, + 'RawMessage.Data': base64.b64encode(raw_message), + } + + self._build_list_params(params, destinations, + 'Destinations.member') + + return self._make_request('SendRawEmail', params) + + def list_verified_email_addresses(self): + """Fetch a list of the email addresses that have been verified. + + :rtype: dict + :returns: A ListVerifiedEmailAddressesResponse structure. Note that + keys must be unicode strings. + """ + return self._make_request('ListVerifiedEmailAddresses') + + def get_send_quota(self): + """Fetches the user's current activity limits. + + :rtype: dict + :returns: A GetSendQuotaResponse structure. Note that keys must be + unicode strings. + """ + return self._make_request('GetSendQuota') + + def get_send_statistics(self): + """Fetches the user's sending statistics. The result is a list of data + points, representing the last two weeks of sending activity. + + Each data point in the list contains statistics for a 15-minute + interval. + + :rtype: dict + :returns: A GetSendStatisticsResponse structure. Note that keys must be + unicode strings. + """ + return self._make_request('GetSendStatistics') + + def delete_verified_email_address(self, email_address): + """Deletes the specified email address from the list of verified + addresses. + + :type email_adddress: string + :param email_address: The email address to be removed from the list of + verified addreses. + + :rtype: dict + :returns: A DeleteVerifiedEmailAddressResponse structure. Note that + keys must be unicode strings. + """ + return self._make_request('DeleteVerifiedEmailAddress', { + 'EmailAddress': email_address, + }) + + def verify_email_address(self, email_address): + """Verifies an email address. This action causes a confirmation email + message to be sent to the specified address. + + :type email_adddress: string + :param email_address: The email address to be verified. + + :rtype: dict + :returns: A VerifyEmailAddressResponse structure. Note that keys must + be unicode strings. + """ + return self._make_request('VerifyEmailAddress', { + 'EmailAddress': email_address, + }) + diff --git a/backup/src/boto/sns/__init__.py b/backup/src/boto/sns/__init__.py new file mode 100644 index 0000000..9c5a7d7 --- /dev/null +++ b/backup/src/boto/sns/__init__.py @@ -0,0 +1,25 @@ +# Copyright (c) 2010-2011 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010-2011, Eucalyptus Systems, Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# this is here for backward compatibility +# originally, the SNSConnection class was defined here +from connection import SNSConnection diff --git a/backup/src/boto/sns/connection.py b/backup/src/boto/sns/connection.py new file mode 100644 index 0000000..2a49adb --- /dev/null +++ b/backup/src/boto/sns/connection.py @@ -0,0 +1,398 @@ +# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSQueryConnection +from boto.sdb.regioninfo import SDBRegionInfo +import boto +import uuid +try: + import json +except ImportError: + import simplejson as json + +#boto.set_stream_logger('sns') + +class SNSConnection(AWSQueryConnection): + + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'sns.us-east-1.amazonaws.com' + APIVersion = '2010-03-31' + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/', converter=None): + if not region: + region = SDBRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + + def _required_auth_capability(self): + return ['sns'] + + def get_all_topics(self, next_token=None): + """ + :type next_token: string + :param next_token: Token returned by the previous call to + this method. + + """ + params = {'ContentType' : 'JSON'} + if next_token: + params['NextToken'] = next_token + response = self.make_request('ListTopics', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_topic_attributes(self, topic): + """ + Get attributes of a Topic + + :type topic: string + :param topic: The ARN of the topic. + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic} + response = self.make_request('GetTopicAttributes', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def add_permission(self, topic, label, account_ids, actions): + """ + Adds a statement to a topic's access control policy, granting + access for the specified AWS accounts to the specified actions. + + :type topic: string + :param topic: The ARN of the topic. + + :type label: string + :param label: A unique identifier for the new policy statement. + + :type account_ids: list of strings + :param account_ids: The AWS account ids of the users who will be + give access to the specified actions. + + :type actions: list of strings + :param actions: The actions you want to allow for each of the + specified principal(s). + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic, + 'Label' : label} + self.build_list_params(params, account_ids, 'AWSAccountId') + self.build_list_params(params, actions, 'ActionName') + response = self.make_request('AddPermission', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def remove_permission(self, topic, label): + """ + Removes a statement from a topic's access control policy. + + :type topic: string + :param topic: The ARN of the topic. + + :type label: string + :param label: A unique identifier for the policy statement + to be removed. + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic, + 'Label' : label} + response = self.make_request('RemovePermission', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def create_topic(self, topic): + """ + Create a new Topic. + + :type topic: string + :param topic: The name of the new topic. + + """ + params = {'ContentType' : 'JSON', + 'Name' : topic} + response = self.make_request('CreateTopic', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def delete_topic(self, topic): + """ + Delete an existing topic + + :type topic: string + :param topic: The ARN of the topic + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic} + response = self.make_request('DeleteTopic', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + + + def publish(self, topic, message, subject=None): + """ + Get properties of a Topic + + :type topic: string + :param topic: The ARN of the new topic. + + :type message: string + :param message: The message you want to send to the topic. + Messages must be UTF-8 encoded strings and + be at most 4KB in size. + + :type subject: string + :param subject: Optional parameter to be used as the "Subject" + line of the email notifications. + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic, + 'Message' : message} + if subject: + params['Subject'] = subject + response = self.make_request('Publish', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def subscribe(self, topic, protocol, endpoint): + """ + Subscribe to a Topic. + + :type topic: string + :param topic: The name of the new topic. + + :type protocol: string + :param protocol: The protocol used to communicate with + the subscriber. Current choices are: + email|email-json|http|https|sqs + + :type endpoint: string + :param endpoint: The location of the endpoint for + the subscriber. + * For email, this would be a valid email address + * For email-json, this would be a valid email address + * For http, this would be a URL beginning with http + * For https, this would be a URL beginning with https + * For sqs, this would be the ARN of an SQS Queue + + :rtype: :class:`boto.sdb.domain.Domain` object + :return: The newly created domain + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic, + 'Protocol' : protocol, + 'Endpoint' : endpoint} + response = self.make_request('Subscribe', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def subscribe_sqs_queue(self, topic, queue): + """ + Subscribe an SQS queue to a topic. + + This is convenience method that handles most of the complexity involved + in using ans SQS queue as an endpoint for an SNS topic. To achieve this + the following operations are performed: + + * The correct ARN is constructed for the SQS queue and that ARN is + then subscribed to the topic. + * A JSON policy document is contructed that grants permission to + the SNS topic to send messages to the SQS queue. + * This JSON policy is then associated with the SQS queue using + the queue's set_attribute method. If the queue already has + a policy associated with it, this process will add a Statement to + that policy. If no policy exists, a new policy will be created. + + :type topic: string + :param topic: The name of the new topic. + + :type queue: A boto Queue object + :param queue: The queue you wish to subscribe to the SNS Topic. + """ + t = queue.id.split('/') + q_arn = 'arn:aws:sqs:%s:%s:%s' % (queue.connection.region.name, + t[1], t[2]) + resp = self.subscribe(topic, 'sqs', q_arn) + policy = queue.get_attributes('Policy') + if 'Version' not in policy: + policy['Version'] = '2008-10-17' + if 'Statement' not in policy: + policy['Statement'] = [] + statement = {'Action' : 'SQS:SendMessage', + 'Effect' : 'Allow', + 'Principal' : {'AWS' : '*'}, + 'Resource' : q_arn, + 'Sid' : str(uuid.uuid4()), + 'Condition' : {'StringLike' : {'aws:SourceArn' : topic}}} + policy['Statement'].append(statement) + queue.set_attribute('Policy', json.dumps(policy)) + return resp + + def confirm_subscription(self, topic, token, + authenticate_on_unsubscribe=False): + """ + Get properties of a Topic + + :type topic: string + :param topic: The ARN of the new topic. + + :type token: string + :param token: Short-lived token sent to and endpoint during + the Subscribe operation. + + :type authenticate_on_unsubscribe: bool + :param authenticate_on_unsubscribe: Optional parameter indicating + that you wish to disable + unauthenticated unsubscription + of the subscription. + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic, + 'Token' : token} + if authenticate_on_unsubscribe: + params['AuthenticateOnUnsubscribe'] = 'true' + response = self.make_request('ConfirmSubscription', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def unsubscribe(self, subscription): + """ + Allows endpoint owner to delete subscription. + Confirmation message will be delivered. + + :type subscription: string + :param subscription: The ARN of the subscription to be deleted. + + """ + params = {'ContentType' : 'JSON', + 'SubscriptionArn' : subscription} + response = self.make_request('Unsubscribe', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_all_subscriptions(self, next_token=None): + """ + Get list of all subscriptions. + + :type next_token: string + :param next_token: Token returned by the previous call to + this method. + + """ + params = {'ContentType' : 'JSON'} + if next_token: + params['NextToken'] = next_token + response = self.make_request('ListSubscriptions', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + + def get_all_subscriptions_by_topic(self, topic, next_token=None): + """ + Get list of all subscriptions to a specific topic. + + :type topic: string + :param topic: The ARN of the topic for which you wish to + find subscriptions. + + :type next_token: string + :param next_token: Token returned by the previous call to + this method. + + """ + params = {'ContentType' : 'JSON', + 'TopicArn' : topic} + if next_token: + params['NextToken'] = next_token + response = self.make_request('ListSubscriptions', params, '/', 'GET') + body = response.read() + if response.status == 200: + return json.loads(body) + else: + boto.log.error('%s %s' % (response.status, response.reason)) + boto.log.error('%s' % body) + raise self.ResponseError(response.status, response.reason, body) + diff --git a/backup/src/boto/sqs/__init__.py b/backup/src/boto/sqs/__init__.py new file mode 100644 index 0000000..463c42c --- /dev/null +++ b/backup/src/boto/sqs/__init__.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from regioninfo import SQSRegionInfo + +def regions(): + """ + Get all available regions for the SQS service. + + :rtype: list + :return: A list of :class:`boto.ec2.regioninfo.RegionInfo` + """ + return [SQSRegionInfo(name='us-east-1', + endpoint='queue.amazonaws.com'), + SQSRegionInfo(name='eu-west-1', + endpoint='eu-west-1.queue.amazonaws.com'), + SQSRegionInfo(name='us-west-1', + endpoint='us-west-1.queue.amazonaws.com'), + SQSRegionInfo(name='ap-southeast-1', + endpoint='ap-southeast-1.queue.amazonaws.com') + ] + +def connect_to_region(region_name): + for region in regions(): + if region.name == region_name: + return region.connect() + return None diff --git a/backup/src/boto/sqs/attributes.py b/backup/src/boto/sqs/attributes.py new file mode 100644 index 0000000..26c7204 --- /dev/null +++ b/backup/src/boto/sqs/attributes.py @@ -0,0 +1,46 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Attribute Name/Value set +""" + +class Attributes(dict): + + def __init__(self, parent): + self.parent = parent + self.current_key = None + self.current_value = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'Attribute': + self[self.current_key] = self.current_value + elif name == 'Name': + self.current_key = value + elif name == 'Value': + self.current_value = value + else: + setattr(self, name, value) + + diff --git a/backup/src/boto/sqs/connection.py b/backup/src/boto/sqs/connection.py new file mode 100644 index 0000000..240fc72 --- /dev/null +++ b/backup/src/boto/sqs/connection.py @@ -0,0 +1,288 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.connection import AWSQueryConnection +from boto.sqs.regioninfo import SQSRegionInfo +from boto.sqs.queue import Queue +from boto.sqs.message import Message +from boto.sqs.attributes import Attributes +from boto.exception import SQSError + + +class SQSConnection(AWSQueryConnection): + """ + A Connection to the SQS Service. + """ + DefaultRegionName = 'us-east-1' + DefaultRegionEndpoint = 'queue.amazonaws.com' + APIVersion = '2009-02-01' + DefaultContentType = 'text/plain' + ResponseError = SQSError + + def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, + is_secure=True, port=None, proxy=None, proxy_port=None, + proxy_user=None, proxy_pass=None, debug=0, + https_connection_factory=None, region=None, path='/'): + if not region: + region = SQSRegionInfo(self, self.DefaultRegionName, self.DefaultRegionEndpoint) + self.region = region + AWSQueryConnection.__init__(self, aws_access_key_id, aws_secret_access_key, + is_secure, port, proxy, proxy_port, proxy_user, proxy_pass, + self.region.endpoint, debug, https_connection_factory, path) + + def _required_auth_capability(self): + return ['sqs'] + + def create_queue(self, queue_name, visibility_timeout=None): + """ + Create an SQS Queue. + + :type queue_name: str or unicode + :param queue_name: The name of the new queue. Names are scoped to an account and need to + be unique within that account. Calling this method on an existing + queue name will not return an error from SQS unless the value for + visibility_timeout is different than the value of the existing queue + of that name. This is still an expensive operation, though, and not + the preferred way to check for the existence of a queue. See the + :func:`boto.sqs.connection.SQSConnection.lookup` method. + + :type visibility_timeout: int + :param visibility_timeout: The default visibility timeout for all messages written in the + queue. This can be overridden on a per-message. + + :rtype: :class:`boto.sqs.queue.Queue` + :return: The newly created queue. + + """ + params = {'QueueName': queue_name} + if visibility_timeout: + params['DefaultVisibilityTimeout'] = '%d' % (visibility_timeout,) + return self.get_object('CreateQueue', params, Queue) + + def delete_queue(self, queue, force_deletion=False): + """ + Delete an SQS Queue. + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type force_deletion: Boolean + :param force_deletion: Normally, SQS will not delete a queue that contains messages. + However, if the force_deletion argument is True, the + queue will be deleted regardless of whether there are messages in + the queue or not. USE WITH CAUTION. This will delete all + messages in the queue as well. + + :rtype: bool + :return: True if the command succeeded, False otherwise + """ + return self.get_status('DeleteQueue', None, queue.id) + + def get_queue_attributes(self, queue, attribute='All'): + """ + Gets one or all attributes of a Queue + + :type queue: A Queue object + :param queue: The SQS queue to be deleted + + :type attribute: str + :type attribute: The specific attribute requested. If not supplied, the default + is to return all attributes. Valid attributes are: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + + :rtype: :class:`boto.sqs.attributes.Attributes` + :return: An Attributes object containing request value(s). + """ + params = {'AttributeName' : attribute} + return self.get_object('GetQueueAttributes', params, Attributes, queue.id) + + def set_queue_attribute(self, queue, attribute, value): + params = {'Attribute.Name' : attribute, 'Attribute.Value' : value} + return self.get_status('SetQueueAttributes', params, queue.id) + + def receive_message(self, queue, number_messages=1, visibility_timeout=None, + attributes=None): + """ + Read messages from an SQS Queue. + + :type queue: A Queue object + :param queue: The Queue from which messages are read. + + :type number_messages: int + :param number_messages: The maximum number of messages to read (default=1) + + :type visibility_timeout: int + :param visibility_timeout: The number of seconds the message should remain invisible + to other queue readers (default=None which uses the Queues default) + + :type attributes: str + :param attributes: The name of additional attribute to return with response + or All if you want all attributes. The default is to + return no additional attributes. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + + :rtype: list + :return: A list of :class:`boto.sqs.message.Message` objects. + """ + params = {'MaxNumberOfMessages' : number_messages} + if visibility_timeout: + params['VisibilityTimeout'] = visibility_timeout + if attributes: + self.build_list_params(params, attributes, 'AttributeName') + return self.get_list('ReceiveMessage', params, [('Message', queue.message_class)], + queue.id, queue) + + def delete_message(self, queue, message): + """ + Delete a message from a queue. + + :type queue: A :class:`boto.sqs.queue.Queue` object + :param queue: The Queue from which messages are read. + + :type message: A :class:`boto.sqs.message.Message` object + :param message: The Message to be deleted + + :rtype: bool + :return: True if successful, False otherwise. + """ + params = {'ReceiptHandle' : message.receipt_handle} + return self.get_status('DeleteMessage', params, queue.id) + + def delete_message_from_handle(self, queue, receipt_handle): + """ + Delete a message from a queue, given a receipt handle. + + :type queue: A :class:`boto.sqs.queue.Queue` object + :param queue: The Queue from which messages are read. + + :type receipt_handle: str + :param receipt_handle: The receipt handle for the message + + :rtype: bool + :return: True if successful, False otherwise. + """ + params = {'ReceiptHandle' : receipt_handle} + return self.get_status('DeleteMessage', params, queue.id) + + def send_message(self, queue, message_content): + params = {'MessageBody' : message_content} + return self.get_object('SendMessage', params, Message, queue.id, verb='POST') + + def change_message_visibility(self, queue, receipt_handle, visibility_timeout): + """ + Extends the read lock timeout for the specified message from the specified queue + to the specified value. + + :type queue: A :class:`boto.sqs.queue.Queue` object + :param queue: The Queue from which messages are read. + + :type receipt_handle: str + :param queue: The receipt handle associated with the message whose + visibility timeout will be changed. + + :type visibility_timeout: int + :param visibility_timeout: The new value of the message's visibility timeout + in seconds. + """ + params = {'ReceiptHandle' : receipt_handle, + 'VisibilityTimeout' : visibility_timeout} + return self.get_status('ChangeMessageVisibility', params, queue.id) + + def get_all_queues(self, prefix=''): + params = {} + if prefix: + params['QueueNamePrefix'] = prefix + return self.get_list('ListQueues', params, [('QueueUrl', Queue)]) + + def get_queue(self, queue_name): + rs = self.get_all_queues(queue_name) + for q in rs: + if q.url.endswith(queue_name): + return q + return None + + lookup = get_queue + + # + # Permissions methods + # + + def add_permission(self, queue, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + params = {'Label': label, + 'AWSAccountId' : aws_account_id, + 'ActionName' : action_name} + return self.get_status('AddPermission', params, queue.id) + + def remove_permission(self, queue, label): + """ + Remove a permission from a queue. + + :type queue: :class:`boto.sqs.queue.Queue` + :param queue: The queue object + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + params = {'Label': label} + return self.get_status('RemovePermission', params, queue.id) + + + + + diff --git a/backup/src/boto/sqs/jsonmessage.py b/backup/src/boto/sqs/jsonmessage.py new file mode 100644 index 0000000..24a3be2 --- /dev/null +++ b/backup/src/boto/sqs/jsonmessage.py @@ -0,0 +1,45 @@ +# Copyright (c) 2006-2008 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +from boto.sqs.message import MHMessage +from boto.exception import SQSDecodeError +import base64 +try: + import json +except ImportError: + import simplejson as json + +class JSONMessage(MHMessage): + """ + Acts like a dictionary but encodes it's data as a Base64 encoded JSON payload. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + value = json.loads(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + + def encode(self, value): + value = json.dumps(value) + return base64.b64encode(value) diff --git a/backup/src/boto/sqs/message.py b/backup/src/boto/sqs/message.py new file mode 100644 index 0000000..8fabd47 --- /dev/null +++ b/backup/src/boto/sqs/message.py @@ -0,0 +1,251 @@ +# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +SQS Message + +A Message represents the data stored in an SQS queue. The rules for what is allowed within an SQS +Message are here: + + http://docs.amazonwebservices.com/AWSSimpleQueueService/2008-01-01/SQSDeveloperGuide/Query_QuerySendMessage.html + +So, at it's simplest level a Message just needs to allow a developer to store bytes in it and get the bytes +back out. However, to allow messages to have richer semantics, the Message class must support the +following interfaces: + +The constructor for the Message class must accept a keyword parameter "queue" which is an instance of a +boto Queue object and represents the queue that the message will be stored in. The default value for +this parameter is None. + +The constructor for the Message class must accept a keyword parameter "body" which represents the +content or body of the message. The format of this parameter will depend on the behavior of the +particular Message subclass. For example, if the Message subclass provides dictionary-like behavior to the +user the body passed to the constructor should be a dict-like object that can be used to populate +the initial state of the message. + +The Message class must provide an encode method that accepts a value of the same type as the body +parameter of the constructor and returns a string of characters that are able to be stored in an +SQS message body (see rules above). + +The Message class must provide a decode method that accepts a string of characters that can be +stored (and probably were stored!) in an SQS message and return an object of a type that is consistent +with the "body" parameter accepted on the class constructor. + +The Message class must provide a __len__ method that will return the size of the encoded message +that would be stored in SQS based on the current state of the Message object. + +The Message class must provide a get_body method that will return the body of the message in the +same format accepted in the constructor of the class. + +The Message class must provide a set_body method that accepts a message body in the same format +accepted by the constructor of the class. This method should alter to the internal state of the +Message object to reflect the state represented in the message body parameter. + +The Message class must provide a get_body_encoded method that returns the current body of the message +in the format in which it would be stored in SQS. +""" + +import base64 +import StringIO +from boto.sqs.attributes import Attributes +from boto.exception import SQSDecodeError + +class RawMessage: + """ + Base class for SQS messages. RawMessage does not encode the message + in any way. Whatever you store in the body of the message is what + will be written to SQS and whatever is returned from SQS is stored + directly into the body of the message. + """ + + def __init__(self, queue=None, body=''): + self.queue = queue + self.set_body(body) + self.id = None + self.receipt_handle = None + self.md5 = None + self.attributes = Attributes(self) + + def __len__(self): + return len(self.encode(self._body)) + + def startElement(self, name, attrs, connection): + if name == 'Attribute': + return self.attributes + return None + + def endElement(self, name, value, connection): + if name == 'Body': + self.set_body(self.decode(value)) + elif name == 'MessageId': + self.id = value + elif name == 'ReceiptHandle': + self.receipt_handle = value + elif name == 'MD5OfMessageBody': + self.md5 = value + else: + setattr(self, name, value) + + def encode(self, value): + """Transform body object into serialized byte array format.""" + return value + + def decode(self, value): + """Transform seralized byte array into any object.""" + return value + + def set_body(self, body): + """Override the current body for this object, using decoded format.""" + self._body = body + + def get_body(self): + return self._body + + def get_body_encoded(self): + """ + This method is really a semi-private method used by the Queue.write + method when writing the contents of the message to SQS. + You probably shouldn't need to call this method in the normal course of events. + """ + return self.encode(self.get_body()) + + def delete(self): + if self.queue: + return self.queue.delete_message(self) + + def change_visibility(self, visibility_timeout): + if self.queue: + self.queue.connection.change_message_visibility(self.queue, + self.receipt_handle, + visibility_timeout) + +class Message(RawMessage): + """ + The default Message class used for SQS queues. This class automatically + encodes/decodes the message body using Base64 encoding to avoid any + illegal characters in the message body. See: + + http://developer.amazonwebservices.com/connect/thread.jspa?messageID=49680%EC%88%90 + + for details on why this is a good idea. The encode/decode is meant to + be transparent to the end-user. + """ + + def encode(self, value): + return base64.b64encode(value) + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return value + +class MHMessage(Message): + """ + The MHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + The encoding/decoding of this is handled automatically and after + the message body has been read, the message instance can be treated + like a mapping object, i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def __init__(self, queue=None, body=None, xml_attrs=None): + if body == None or body == '': + body = {} + Message.__init__(self, queue, body) + + def decode(self, value): + try: + msg = {} + fp = StringIO.StringIO(value) + line = fp.readline() + while line: + delim = line.find(':') + key = line[0:delim] + value = line[delim+1:].strip() + msg[key.strip()] = value.strip() + line = fp.readline() + except: + raise SQSDecodeError('Unable to decode message', self) + return msg + + def encode(self, value): + s = '' + for item in value.items(): + s = s + '%s: %s\n' % (item[0], item[1]) + return s + + def __getitem__(self, key): + if self._body.has_key(key): + return self._body[key] + else: + raise KeyError(key) + + def __setitem__(self, key, value): + self._body[key] = value + self.set_body(self._body) + + def keys(self): + return self._body.keys() + + def values(self): + return self._body.values() + + def items(self): + return self._body.items() + + def has_key(self, key): + return self._body.has_key(key) + + def update(self, d): + self._body.update(d) + self.set_body(self._body) + + def get(self, key, default=None): + return self._body.get(key, default) + +class EncodedMHMessage(MHMessage): + """ + The EncodedMHMessage class provides a message that provides RFC821-like + headers like this: + + HeaderName: HeaderValue + + This variation encodes/decodes the body of the message in base64 automatically. + The message instance can be treated like a mapping object, + i.e. m['HeaderName'] would return 'HeaderValue'. + """ + + def decode(self, value): + try: + value = base64.b64decode(value) + except: + raise SQSDecodeError('Unable to decode message', self) + return MHMessage.decode(self, value) + + def encode(self, value): + value = MHMessage.encode(self, value) + return base64.b64encode(value) + diff --git a/backup/src/boto/sqs/queue.py b/backup/src/boto/sqs/queue.py new file mode 100644 index 0000000..9965e43 --- /dev/null +++ b/backup/src/boto/sqs/queue.py @@ -0,0 +1,414 @@ +# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents an SQS Queue +""" + +import urlparse +from boto.sqs.message import Message + + +class Queue: + + def __init__(self, connection=None, url=None, message_class=Message): + self.connection = connection + self.url = url + self.message_class = message_class + self.visibility_timeout = None + + def _id(self): + if self.url: + val = urlparse.urlparse(self.url)[2] + else: + val = self.url + return val + id = property(_id) + + def _name(self): + if self.url: + val = urlparse.urlparse(self.url)[2].split('/')[2] + else: + val = self.url + return val + name = property(_name) + + def startElement(self, name, attrs, connection): + return None + + def endElement(self, name, value, connection): + if name == 'QueueUrl': + self.url = value + elif name == 'VisibilityTimeout': + self.visibility_timeout = int(value) + else: + setattr(self, name, value) + + def set_message_class(self, message_class): + """ + Set the message class that should be used when instantiating messages read + from the queue. By default, the class boto.sqs.message.Message is used but + this can be overriden with any class that behaves like a message. + + :type message_class: Message-like class + :param message_class: The new Message class + """ + self.message_class = message_class + + def get_attributes(self, attributes='All'): + """ + Retrieves attributes about this queue object and returns + them in an Attribute instance (subclass of a Dictionary). + + :type attributes: string + :param attributes: String containing one of: + ApproximateNumberOfMessages, + ApproximateNumberOfMessagesNotVisible, + VisibilityTimeout, + CreatedTimestamp, + LastModifiedTimestamp, + Policy + :rtype: Attribute object + :return: An Attribute object which is a mapping type holding the + requested name/value pairs + """ + return self.connection.get_queue_attributes(self, attributes) + + def set_attribute(self, attribute, value): + """ + Set a new value for an attribute of the Queue. + + :type attribute: String + :param attribute: The name of the attribute you want to set. The + only valid value at this time is: VisibilityTimeout + :type value: int + :param value: The new value for the attribute. + For VisibilityTimeout the value must be an + integer number of seconds from 0 to 86400. + + :rtype: bool + :return: True if successful, otherwise False. + """ + return self.connection.set_queue_attribute(self, attribute, value) + + def get_timeout(self): + """ + Get the visibility timeout for the queue. + + :rtype: int + :return: The number of seconds as an integer. + """ + a = self.get_attributes('VisibilityTimeout') + return int(a['VisibilityTimeout']) + + def set_timeout(self, visibility_timeout): + """ + Set the visibility timeout for the queue. + + :type visibility_timeout: int + :param visibility_timeout: The desired timeout in seconds + """ + retval = self.set_attribute('VisibilityTimeout', visibility_timeout) + if retval: + self.visibility_timeout = visibility_timeout + return retval + + def add_permission(self, label, aws_account_id, action_name): + """ + Add a permission to a queue. + + :type label: str or unicode + :param label: A unique identification of the permission you are setting. + Maximum of 80 characters ``[0-9a-zA-Z_-]`` + Example, AliceSendMessage + + :type aws_account_id: str or unicode + :param principal_id: The AWS account number of the principal who will be given + permission. The principal must have an AWS account, but + does not need to be signed up for Amazon SQS. For information + about locating the AWS account identification. + + :type action_name: str or unicode + :param action_name: The action. Valid choices are: + \*|SendMessage|ReceiveMessage|DeleteMessage| + ChangeMessageVisibility|GetQueueAttributes + + :rtype: bool + :return: True if successful, False otherwise. + + """ + return self.connection.add_permission(self, label, aws_account_id, action_name) + + def remove_permission(self, label): + """ + Remove a permission from a queue. + + :type label: str or unicode + :param label: The unique label associated with the permission being removed. + + :rtype: bool + :return: True if successful, False otherwise. + """ + return self.connection.remove_permission(self, label) + + def read(self, visibility_timeout=None): + """ + Read a single message from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The timeout for this message in seconds + + :rtype: :class:`boto.sqs.message.Message` + :return: A single message or None if queue is empty + """ + rs = self.get_messages(1, visibility_timeout) + if len(rs) == 1: + return rs[0] + else: + return None + + def write(self, message): + """ + Add a single message to the queue. + + :type message: Message + :param message: The message to be written to the queue + + :rtype: :class:`boto.sqs.message.Message` + :return: The :class:`boto.sqs.message.Message` object that was written. + """ + new_msg = self.connection.send_message(self, message.get_body_encoded()) + message.id = new_msg.id + message.md5 = new_msg.md5 + return message + + def new_message(self, body=''): + """ + Create new message of appropriate class. + + :type body: message body + :param body: The body of the newly created message (optional). + + :rtype: :class:`boto.sqs.message.Message` + :return: A new Message object + """ + m = self.message_class(self, body) + m.queue = self + return m + + # get a variable number of messages, returns a list of messages + def get_messages(self, num_messages=1, visibility_timeout=None, + attributes=None): + """ + Get a variable number of messages. + + :type num_messages: int + :param num_messages: The maximum number of messages to read from the queue. + + :type visibility_timeout: int + :param visibility_timeout: The VisibilityTimeout for the messages read. + + :type attributes: str + :param attributes: The name of additional attribute to return with response + or All if you want all attributes. The default is to + return no additional attributes. Valid values: + All + SenderId + SentTimestamp + ApproximateReceiveCount + ApproximateFirstReceiveTimestamp + + :rtype: list + :return: A list of :class:`boto.sqs.message.Message` objects. + """ + return self.connection.receive_message(self, number_messages=num_messages, + visibility_timeout=visibility_timeout, + attributes=attributes) + + def delete_message(self, message): + """ + Delete a message from the queue. + + :type message: :class:`boto.sqs.message.Message` + :param message: The :class:`boto.sqs.message.Message` object to delete. + + :rtype: bool + :return: True if successful, False otherwise + """ + return self.connection.delete_message(self, message) + + def delete(self): + """ + Delete the queue. + """ + return self.connection.delete_queue(self) + + def clear(self, page_size=10, vtimeout=10): + """Utility function to remove all messages from a queue""" + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + self.delete_message(m) + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def count(self, page_size=10, vtimeout=10): + """ + Utility function to count the number of messages in a queue. + Note: This function now calls GetQueueAttributes to obtain + an 'approximate' count of the number of messages in a queue. + """ + a = self.get_attributes('ApproximateNumberOfMessages') + return int(a['ApproximateNumberOfMessages']) + + def count_slow(self, page_size=10, vtimeout=10): + """ + Deprecated. This is the old 'count' method that actually counts + the messages by reading them all. This gives an accurate count but + is very slow for queues with non-trivial number of messasges. + Instead, use get_attribute('ApproximateNumberOfMessages') to take + advantage of the new SQS capability. This is retained only for + the unit tests. + """ + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + n += 1 + l = self.get_messages(page_size, vtimeout) + return n + + def dump(self, file_name, page_size=10, vtimeout=10, sep='\n'): + """Utility function to dump the messages in a queue to a file + NOTE: Page size must be < 10 else SQS errors""" + fp = open(file_name, 'wb') + n = 0 + l = self.get_messages(page_size, vtimeout) + while l: + for m in l: + fp.write(m.get_body()) + if sep: + fp.write(sep) + n += 1 + l = self.get_messages(page_size, vtimeout) + fp.close() + return n + + def save_to_file(self, fp, sep='\n'): + """ + Read all messages from the queue and persist them to file-like object. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + fp.write(m.get_body()) + if sep: + fp.write(sep) + self.delete_message(m) + m = self.read() + return n + + def save_to_filename(self, file_name, sep='\n'): + """ + Read all messages from the queue and persist them to local file. + Messages are written to the file and the 'sep' string is written + in between messages. Messages are deleted from the queue after + being written to the file. + Returns the number of messages saved. + """ + fp = open(file_name, 'wb') + n = self.save_to_file(fp, sep) + fp.close() + return n + + # for backwards compatibility + save = save_to_filename + + def save_to_s3(self, bucket): + """ + Read all messages from the queue and persist them to S3. + Messages are stored in the S3 bucket using a naming scheme of:: + + / + + Messages are deleted from the queue after being saved to S3. + Returns the number of messages saved. + """ + n = 0 + m = self.read() + while m: + n += 1 + key = bucket.new_key('%s/%s' % (self.id, m.id)) + key.set_contents_from_string(m.get_body()) + self.delete_message(m) + m = self.read() + return n + + def load_from_s3(self, bucket, prefix=None): + """ + Load messages previously saved to S3. + """ + n = 0 + if prefix: + prefix = '%s/' % prefix + else: + prefix = '%s/' % self.id[1:] + rs = bucket.list(prefix=prefix) + for key in rs: + n += 1 + m = self.new_message(key.get_contents_as_string()) + self.write(m) + return n + + def load_from_file(self, fp, sep='\n'): + """Utility function to load messages from a file-like object to a queue""" + n = 0 + body = '' + l = fp.readline() + while l: + if l == sep: + m = Message(self, body) + self.write(m) + n += 1 + print 'writing message %d' % n + body = '' + else: + body = body + l + l = fp.readline() + return n + + def load_from_filename(self, file_name, sep='\n'): + """Utility function to load messages from a local filename to a queue""" + fp = open(file_name, 'rb') + n = self.load_file_file(fp, sep) + fp.close() + return n + + # for backward compatibility + load = load_from_filename + diff --git a/backup/src/boto/sqs/regioninfo.py b/backup/src/boto/sqs/regioninfo.py new file mode 100644 index 0000000..66d6733 --- /dev/null +++ b/backup/src/boto/sqs/regioninfo.py @@ -0,0 +1,32 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +from boto.regioninfo import RegionInfo + +class SQSRegionInfo(RegionInfo): + + def __init__(self, connection=None, name=None, endpoint=None): + from boto.sqs.connection import SQSConnection + RegionInfo.__init__(self, connection, name, endpoint, + SQSConnection) diff --git a/backup/src/boto/storage_uri.py b/backup/src/boto/storage_uri.py new file mode 100644 index 0000000..9c051a4 --- /dev/null +++ b/backup/src/boto/storage_uri.py @@ -0,0 +1,380 @@ +# Copyright 2010 Google Inc. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +import os +from boto.exception import BotoClientError +from boto.exception import InvalidUriError + + +class StorageUri(object): + """ + Base class for representing storage provider-independent bucket and + object name with a shorthand URI-like syntax. + + This is an abstract class: the constructor cannot be called (throws an + exception if you try). + """ + + connection = None + + def __init__(self): + """Uncallable constructor on abstract base StorageUri class. + """ + raise BotoClientError('Attempt to instantiate abstract StorageUri ' + 'class') + + def __repr__(self): + """Returns string representation of URI.""" + return self.uri + + def equals(self, uri): + """Returns true if two URIs are equal.""" + return self.uri == uri.uri + + def check_response(self, resp, level, uri): + if resp is None: + raise InvalidUriError('Attempt to get %s for "%s" failed. This ' + 'probably indicates the URI is invalid' % + (level, uri)) + + def connect(self, access_key_id=None, secret_access_key=None, **kwargs): + """ + Opens a connection to appropriate provider, depending on provider + portion of URI. Requires Credentials defined in boto config file (see + boto/pyami/config.py). + @type storage_uri: StorageUri + @param storage_uri: StorageUri specifying a bucket or a bucket+object + @rtype: L{AWSAuthConnection} + @return: A connection to storage service provider of the given URI. + """ + + if not self.connection: + if self.scheme == 's3': + from boto.s3.connection import S3Connection + self.connection = S3Connection(access_key_id, + secret_access_key, **kwargs) + elif self.scheme == 'gs': + from boto.gs.connection import GSConnection + self.connection = GSConnection(access_key_id, + secret_access_key, **kwargs) + elif self.scheme == 'file': + from boto.file.connection import FileConnection + self.connection = FileConnection(self) + else: + raise InvalidUriError('Unrecognized scheme "%s"' % + self.scheme) + self.connection.debug = self.debug + return self.connection + + def delete_key(self, validate=True, headers=None, version_id=None, + mfa_token=None): + if not self.object_name: + raise InvalidUriError('delete_key on object-less URI (%s)' % + self.uri) + bucket = self.get_bucket(validate, headers) + return bucket.delete_key(self.object_name, headers, version_id, + mfa_token) + + def get_all_keys(self, validate=True, headers=None): + bucket = self.get_bucket(validate, headers) + return bucket.get_all_keys(headers) + + def get_bucket(self, validate=True, headers=None): + if self.bucket_name is None: + raise InvalidUriError('get_bucket on bucket-less URI (%s)' % + self.uri) + conn = self.connect() + bucket = conn.get_bucket(self.bucket_name, validate, headers) + self.check_response(bucket, 'bucket', self.uri) + return bucket + + def get_key(self, validate=True, headers=None, version_id=None): + if not self.object_name: + raise InvalidUriError('get_key on object-less URI (%s)' % self.uri) + bucket = self.get_bucket(validate, headers) + key = bucket.get_key(self.object_name, headers, version_id) + self.check_response(key, 'key', self.uri) + return key + + def new_key(self, validate=True, headers=None): + if not self.object_name: + raise InvalidUriError('new_key on object-less URI (%s)' % self.uri) + bucket = self.get_bucket(validate, headers) + return bucket.new_key(self.object_name) + + def get_contents_as_string(self, validate=True, headers=None, cb=None, + num_cb=10, torrent=False, version_id=None): + if not self.object_name: + raise InvalidUriError('get_contents_as_string on object-less URI ' + '(%s)' % self.uri) + key = self.get_key(validate, headers) + self.check_response(key, 'key', self.uri) + return key.get_contents_as_string(headers, cb, num_cb, torrent, + version_id) + + def acl_class(self): + if self.bucket_name is None: + raise InvalidUriError('acl_class on bucket-less URI (%s)' % + self.uri) + conn = self.connect() + acl_class = conn.provider.acl_class + self.check_response(acl_class, 'acl_class', self.uri) + return acl_class + + def canned_acls(self): + if self.bucket_name is None: + raise InvalidUriError('canned_acls on bucket-less URI (%s)' % + self.uri) + conn = self.connect() + canned_acls = conn.provider.canned_acls + self.check_response(canned_acls, 'canned_acls', self.uri) + return canned_acls + + +class BucketStorageUri(StorageUri): + """ + StorageUri subclass that handles bucket storage providers. + Callers should instantiate this class by calling boto.storage_uri(). + """ + + def __init__(self, scheme, bucket_name=None, object_name=None, + debug=0): + """Instantiate a BucketStorageUri from scheme,bucket,object tuple. + + @type scheme: string + @param scheme: URI scheme naming the storage provider (gs, s3, etc.) + @type bucket_name: string + @param bucket_name: bucket name + @type object_name: string + @param object_name: object name + @type debug: int + @param debug: debug level to pass in to connection (range 0..2) + + After instantiation the components are available in the following + fields: uri, scheme, bucket_name, object_name. + """ + + self.scheme = scheme + self.bucket_name = bucket_name + self.object_name = object_name + if self.bucket_name and self.object_name: + self.uri = ('%s://%s/%s' % (self.scheme, self.bucket_name, + self.object_name)) + elif self.bucket_name: + self.uri = ('%s://%s/' % (self.scheme, self.bucket_name)) + else: + self.uri = ('%s://' % self.scheme) + self.debug = debug + + def clone_replace_name(self, new_name): + """Instantiate a BucketStorageUri from the current BucketStorageUri, + but replacing the object_name. + + @type new_name: string + @param new_name: new object name + """ + if not self.bucket_name: + raise InvalidUriError('clone_replace_name() on bucket-less URI %s' % + self.uri) + return BucketStorageUri(self.scheme, self.bucket_name, new_name, + self.debug) + + def get_acl(self, validate=True, headers=None, version_id=None): + if not self.bucket_name: + raise InvalidUriError('get_acl on bucket-less URI (%s)' % self.uri) + bucket = self.get_bucket(validate, headers) + # This works for both bucket- and object- level ACLs (former passes + # key_name=None): + acl = bucket.get_acl(self.object_name, headers, version_id) + self.check_response(acl, 'acl', self.uri) + return acl + + def add_group_email_grant(self, permission, email_address, recursive=False, + validate=True, headers=None): + if self.scheme != 'gs': + raise ValueError('add_group_email_grant() not supported for %s ' + 'URIs.' % self.scheme) + if self.object_name: + if recursive: + raise ValueError('add_group_email_grant() on key-ful URI cannot ' + 'specify recursive=True') + key = self.get_key(validate, headers) + self.check_response(key, 'key', self.uri) + key.add_group_email_grant(permission, email_address, headers) + elif self.bucket_name: + bucket = self.get_bucket(validate, headers) + bucket.add_group_email_grant(permission, email_address, recursive, + headers) + else: + raise InvalidUriError('add_group_email_grant() on bucket-less URI %s' % + self.uri) + + def add_email_grant(self, permission, email_address, recursive=False, + validate=True, headers=None): + if not self.bucket_name: + raise InvalidUriError('add_email_grant on bucket-less URI (%s)' % + self.uri) + if not self.object_name: + bucket = self.get_bucket(validate, headers) + bucket.add_email_grant(permission, email_address, recursive, + headers) + else: + key = self.get_key(validate, headers) + self.check_response(key, 'key', self.uri) + key.add_email_grant(permission, email_address) + + def add_user_grant(self, permission, user_id, recursive=False, + validate=True, headers=None): + if not self.bucket_name: + raise InvalidUriError('add_user_grant on bucket-less URI (%s)' % + self.uri) + if not self.object_name: + bucket = self.get_bucket(validate, headers) + bucket.add_user_grant(permission, user_id, recursive, headers) + else: + key = self.get_key(validate, headers) + self.check_response(key, 'key', self.uri) + key.add_user_grant(permission, user_id) + + def list_grants(self, headers=None): + if not self.bucket_name: + raise InvalidUriError('list_grants on bucket-less URI (%s)' % + self.uri) + bucket = self.get_bucket(headers) + return bucket.list_grants(headers) + + def names_container(self): + """Returns True if this URI names a bucket (vs. an object). + """ + return not self.object_name + + def names_singleton(self): + """Returns True if this URI names an object (vs. a bucket). + """ + return self.object_name + + def is_file_uri(self): + return False + + def is_cloud_uri(self): + return True + + def create_bucket(self, headers=None, location='', policy=None): + if self.bucket_name is None: + raise InvalidUriError('create_bucket on bucket-less URI (%s)' % + self.uri) + conn = self.connect() + return conn.create_bucket(self.bucket_name, headers, location, policy) + + def delete_bucket(self, headers=None): + if self.bucket_name is None: + raise InvalidUriError('delete_bucket on bucket-less URI (%s)' % + self.uri) + conn = self.connect() + return conn.delete_bucket(self.bucket_name, headers) + + def get_all_buckets(self, headers=None): + conn = self.connect() + return conn.get_all_buckets(headers) + + def get_provider(self): + conn = self.connect() + provider = conn.provider + self.check_response(provider, 'provider', self.uri) + return provider + + def set_acl(self, acl_or_str, key_name='', validate=True, headers=None, + version_id=None): + if not self.bucket_name: + raise InvalidUriError('set_acl on bucket-less URI (%s)' % + self.uri) + self.get_bucket(validate, headers).set_acl(acl_or_str, key_name, + headers, version_id) + + def set_canned_acl(self, acl_str, validate=True, headers=None, + version_id=None): + if not self.object_name: + raise InvalidUriError('set_canned_acl on object-less URI (%s)' % + self.uri) + key = self.get_key(validate, headers) + self.check_response(key, 'key', self.uri) + key.set_canned_acl(acl_str, headers, version_id) + + def set_contents_from_string(self, s, headers=None, replace=True, + cb=None, num_cb=10, policy=None, md5=None, + reduced_redundancy=False): + key = self.new_key(headers=headers) + key.set_contents_from_string(s, headers, replace, cb, num_cb, policy, + md5, reduced_redundancy) + + + +class FileStorageUri(StorageUri): + """ + StorageUri subclass that handles files in the local file system. + Callers should instantiate this class by calling boto.storage_uri(). + + See file/README about how we map StorageUri operations onto a file system. + """ + + def __init__(self, object_name, debug): + """Instantiate a FileStorageUri from a path name. + + @type object_name: string + @param object_name: object name + @type debug: boolean + @param debug: whether to enable debugging on this StorageUri + + After instantiation the components are available in the following + fields: uri, scheme, bucket_name (always blank for this "anonymous" + bucket), object_name. + """ + + self.scheme = 'file' + self.bucket_name = '' + self.object_name = object_name + self.uri = 'file://' + object_name + self.debug = debug + + def clone_replace_name(self, new_name): + """Instantiate a FileStorageUri from the current FileStorageUri, + but replacing the object_name. + + @type new_name: string + @param new_name: new object name + """ + return FileStorageUri(new_name, self.debug) + + def names_container(self): + """Returns True if this URI names a directory. + """ + return os.path.isdir(self.object_name) + + def names_singleton(self): + """Returns True if this URI names a file. + """ + return os.path.isfile(self.object_name) + + def is_file_uri(self): + return True + + def is_cloud_uri(self): + return False diff --git a/backup/src/boto/utils.py b/backup/src/boto/utils.py new file mode 100644 index 0000000..6bad25d --- /dev/null +++ b/backup/src/boto/utils.py @@ -0,0 +1,607 @@ +# Copyright (c) 2006-2010 Mitch Garnaat http://garnaat.org/ +# Copyright (c) 2010, Eucalyptus Systems, Inc. +# All rights reserved. +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +# +# Parts of this code were copied or derived from sample code supplied by AWS. +# The following notice applies to that code. +# +# This software code is made available "AS IS" without warranties of any +# kind. You may copy, display, modify and redistribute the software +# code either by itself or as incorporated into your code; provided that +# you do not remove any proprietary notices. Your use of this software +# code is at your own risk and you waive any claim against Amazon +# Digital Services, Inc. or its affiliates with respect to your use of +# this software code. (c) 2006 Amazon Digital Services, Inc. or its +# affiliates. + +""" +Some handy utility functions used by several classes. +""" + +import urllib +import urllib2 +import imp +import subprocess +import StringIO +import time +import logging.handlers +import boto +import tempfile +import smtplib +import datetime +from email.MIMEMultipart import MIMEMultipart +from email.MIMEBase import MIMEBase +from email.MIMEText import MIMEText +from email.Utils import formatdate +from email import Encoders + +try: + import hashlib + _hashfn = hashlib.sha512 +except ImportError: + import md5 + _hashfn = md5.md5 + +# List of Query String Arguments of Interest +qsa_of_interest = ['acl', 'location', 'logging', 'partNumber', 'policy', + 'requestPayment', 'torrent', 'versioning', 'versionId', + 'versions', 'website', 'uploads', 'uploadId', + 'response-content-type', 'response-content-language', + 'response-expires', 'reponse-cache-control', + 'response-content-disposition', + 'response-content-encoding'] + +# generates the aws canonical string for the given parameters +def canonical_string(method, path, headers, expires=None, + provider=None): + if not provider: + provider = boto.provider.get_default() + interesting_headers = {} + for key in headers: + lk = key.lower() + if headers[key] != None and (lk in ['content-md5', 'content-type', 'date'] or + lk.startswith(provider.header_prefix)): + interesting_headers[lk] = headers[key].strip() + + # these keys get empty strings if they don't exist + if not interesting_headers.has_key('content-type'): + interesting_headers['content-type'] = '' + if not interesting_headers.has_key('content-md5'): + interesting_headers['content-md5'] = '' + + # just in case someone used this. it's not necessary in this lib. + if interesting_headers.has_key(provider.date_header): + interesting_headers['date'] = '' + + # if you're using expires for query string auth, then it trumps date + # (and provider.date_header) + if expires: + interesting_headers['date'] = str(expires) + + sorted_header_keys = interesting_headers.keys() + sorted_header_keys.sort() + + buf = "%s\n" % method + for key in sorted_header_keys: + val = interesting_headers[key] + if key.startswith(provider.header_prefix): + buf += "%s:%s\n" % (key, val) + else: + buf += "%s\n" % val + + # don't include anything after the first ? in the resource... + # unless it is one of the QSA of interest, defined above + t = path.split('?') + buf += t[0] + + if len(t) > 1: + qsa = t[1].split('&') + qsa = [ a.split('=') for a in qsa] + qsa = [ a for a in qsa if a[0] in qsa_of_interest ] + if len(qsa) > 0: + qsa.sort(cmp=lambda x,y:cmp(x[0], y[0])) + qsa = [ '='.join(a) for a in qsa ] + buf += '?' + buf += '&'.join(qsa) + + return buf + +def merge_meta(headers, metadata, provider=None): + if not provider: + provider = boto.provider.get_default() + metadata_prefix = provider.metadata_prefix + final_headers = headers.copy() + for k in metadata.keys(): + if k.lower() in ['cache-control', 'content-md5', 'content-type', + 'content-encoding', 'content-disposition', + 'date', 'expires']: + final_headers[k] = metadata[k] + else: + final_headers[metadata_prefix + k] = metadata[k] + + return final_headers + +def get_aws_metadata(headers, provider=None): + if not provider: + provider = boto.provider.get_default() + metadata_prefix = provider.metadata_prefix + metadata = {} + for hkey in headers.keys(): + if hkey.lower().startswith(metadata_prefix): + val = urllib.unquote_plus(headers[hkey]) + metadata[hkey[len(metadata_prefix):]] = unicode(val, 'utf-8') + del headers[hkey] + return metadata + +def retry_url(url, retry_on_404=True): + for i in range(0, 10): + try: + req = urllib2.Request(url) + resp = urllib2.urlopen(req) + return resp.read() + except urllib2.HTTPError, e: + # in 2.6 you use getcode(), in 2.5 and earlier you use code + if hasattr(e, 'getcode'): + code = e.getcode() + else: + code = e.code + if code == 404 and not retry_on_404: + return '' + except: + pass + boto.log.exception('Caught exception reading instance data') + time.sleep(2**i) + boto.log.error('Unable to read instance data, giving up') + return '' + +def _get_instance_metadata(url): + d = {} + data = retry_url(url) + if data: + fields = data.split('\n') + for field in fields: + if field.endswith('/'): + d[field[0:-1]] = _get_instance_metadata(url + field) + else: + p = field.find('=') + if p > 0: + key = field[p+1:] + resource = field[0:p] + '/openssh-key' + else: + key = resource = field + val = retry_url(url + resource) + p = val.find('\n') + if p > 0: + val = val.split('\n') + d[key] = val + return d + +def get_instance_metadata(version='latest'): + """ + Returns the instance metadata as a nested Python dictionary. + Simple values (e.g. local_hostname, hostname, etc.) will be + stored as string values. Values such as ancestor-ami-ids will + be stored in the dict as a list of string values. More complex + fields such as public-keys and will be stored as nested dicts. + """ + url = 'http://169.254.169.254/%s/meta-data/' % version + return _get_instance_metadata(url) + +def get_instance_userdata(version='latest', sep=None): + url = 'http://169.254.169.254/%s/user-data' % version + user_data = retry_url(url, retry_on_404=False) + if user_data: + if sep: + l = user_data.split(sep) + user_data = {} + for nvpair in l: + t = nvpair.split('=') + user_data[t[0].strip()] = t[1].strip() + return user_data + +ISO8601 = '%Y-%m-%dT%H:%M:%SZ' + +def get_ts(ts=None): + if not ts: + ts = time.gmtime() + return time.strftime(ISO8601, ts) + +def parse_ts(ts): + return datetime.datetime.strptime(ts, ISO8601) + +def find_class(module_name, class_name=None): + if class_name: + module_name = "%s.%s" % (module_name, class_name) + modules = module_name.split('.') + c = None + + try: + for m in modules[1:]: + if c: + c = getattr(c, m) + else: + c = getattr(__import__(".".join(modules[0:-1])), m) + return c + except: + return None + +def update_dme(username, password, dme_id, ip_address): + """ + Update your Dynamic DNS record with DNSMadeEasy.com + """ + dme_url = 'https://www.dnsmadeeasy.com/servlet/updateip' + dme_url += '?username=%s&password=%s&id=%s&ip=%s' + s = urllib2.urlopen(dme_url % (username, password, dme_id, ip_address)) + return s.read() + +def fetch_file(uri, file=None, username=None, password=None): + """ + Fetch a file based on the URI provided. If you do not pass in a file pointer + a tempfile.NamedTemporaryFile, or None if the file could not be + retrieved is returned. + The URI can be either an HTTP url, or "s3://bucket_name/key_name" + """ + boto.log.info('Fetching %s' % uri) + if file == None: + file = tempfile.NamedTemporaryFile() + try: + if uri.startswith('s3://'): + bucket_name, key_name = uri[len('s3://'):].split('/', 1) + c = boto.connect_s3(aws_access_key_id=username, aws_secret_access_key=password) + bucket = c.get_bucket(bucket_name) + key = bucket.get_key(key_name) + key.get_contents_to_file(file) + else: + if username and password: + passman = urllib2.HTTPPasswordMgrWithDefaultRealm() + passman.add_password(None, uri, username, password) + authhandler = urllib2.HTTPBasicAuthHandler(passman) + opener = urllib2.build_opener(authhandler) + urllib2.install_opener(opener) + s = urllib2.urlopen(uri) + file.write(s.read()) + file.seek(0) + except: + raise + boto.log.exception('Problem Retrieving file: %s' % uri) + file = None + return file + +class ShellCommand(object): + + def __init__(self, command, wait=True, fail_fast=False, cwd = None): + self.exit_code = 0 + self.command = command + self.log_fp = StringIO.StringIO() + self.wait = wait + self.fail_fast = fail_fast + self.run(cwd = cwd) + + def run(self, cwd=None): + boto.log.info('running:%s' % self.command) + self.process = subprocess.Popen(self.command, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=cwd) + if(self.wait): + while self.process.poll() == None: + time.sleep(1) + t = self.process.communicate() + self.log_fp.write(t[0]) + self.log_fp.write(t[1]) + boto.log.info(self.log_fp.getvalue()) + self.exit_code = self.process.returncode + + if self.fail_fast and self.exit_code != 0: + raise Exception("Command " + self.command + " failed with status " + self.exit_code) + + return self.exit_code + + def setReadOnly(self, value): + raise AttributeError + + def getStatus(self): + return self.exit_code + + status = property(getStatus, setReadOnly, None, 'The exit code for the command') + + def getOutput(self): + return self.log_fp.getvalue() + + output = property(getOutput, setReadOnly, None, 'The STDIN and STDERR output of the command') + +class AuthSMTPHandler(logging.handlers.SMTPHandler): + """ + This class extends the SMTPHandler in the standard Python logging module + to accept a username and password on the constructor and to then use those + credentials to authenticate with the SMTP server. To use this, you could + add something like this in your boto config file: + + [handler_hand07] + class=boto.utils.AuthSMTPHandler + level=WARN + formatter=form07 + args=('localhost', 'username', 'password', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') + """ + + def __init__(self, mailhost, username, password, fromaddr, toaddrs, subject): + """ + Initialize the handler. + + We have extended the constructor to accept a username/password + for SMTP authentication. + """ + logging.handlers.SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject) + self.username = username + self.password = password + + def emit(self, record): + """ + Emit a record. + + Format the record and send it to the specified addressees. + It would be really nice if I could add authorization to this class + without having to resort to cut and paste inheritance but, no. + """ + try: + port = self.mailport + if not port: + port = smtplib.SMTP_PORT + smtp = smtplib.SMTP(self.mailhost, port) + smtp.login(self.username, self.password) + msg = self.format(record) + msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % ( + self.fromaddr, + ','.join(self.toaddrs), + self.getSubject(record), + formatdate(), msg) + smtp.sendmail(self.fromaddr, self.toaddrs, msg) + smtp.quit() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handleError(record) + +class LRUCache(dict): + """A dictionary-like object that stores only a certain number of items, and + discards its least recently used item when full. + + >>> cache = LRUCache(3) + >>> cache['A'] = 0 + >>> cache['B'] = 1 + >>> cache['C'] = 2 + >>> len(cache) + 3 + + >>> cache['A'] + 0 + + Adding new items to the cache does not increase its size. Instead, the least + recently used item is dropped: + + >>> cache['D'] = 3 + >>> len(cache) + 3 + >>> 'B' in cache + False + + Iterating over the cache returns the keys, starting with the most recently + used: + + >>> for key in cache: + ... print key + D + A + C + + This code is based on the LRUCache class from Genshi which is based on + Mighty's LRUCache from ``myghtyutils.util``, written + by Mike Bayer and released under the MIT license (Genshi uses the + BSD License). See: + + http://svn.myghty.org/myghtyutils/trunk/lib/myghtyutils/util.py + """ + + class _Item(object): + def __init__(self, key, value): + self.previous = self.next = None + self.key = key + self.value = value + def __repr__(self): + return repr(self.value) + + def __init__(self, capacity): + self._dict = dict() + self.capacity = capacity + self.head = None + self.tail = None + + def __contains__(self, key): + return key in self._dict + + def __iter__(self): + cur = self.head + while cur: + yield cur.key + cur = cur.next + + def __len__(self): + return len(self._dict) + + def __getitem__(self, key): + item = self._dict[key] + self._update_item(item) + return item.value + + def __setitem__(self, key, value): + item = self._dict.get(key) + if item is None: + item = self._Item(key, value) + self._dict[key] = item + self._insert_item(item) + else: + item.value = value + self._update_item(item) + self._manage_size() + + def __repr__(self): + return repr(self._dict) + + def _insert_item(self, item): + item.previous = None + item.next = self.head + if self.head is not None: + self.head.previous = item + else: + self.tail = item + self.head = item + self._manage_size() + + def _manage_size(self): + while len(self._dict) > self.capacity: + del self._dict[self.tail.key] + if self.tail != self.head: + self.tail = self.tail.previous + self.tail.next = None + else: + self.head = self.tail = None + + def _update_item(self, item): + if self.head == item: + return + + previous = item.previous + previous.next = item.next + if item.next is not None: + item.next.previous = previous + else: + self.tail = previous + + item.previous = None + item.next = self.head + self.head.previous = self.head = item + +class Password(object): + """ + Password object that stores itself as SHA512 hashed. + """ + def __init__(self, str=None): + """ + Load the string from an initial value, this should be the raw SHA512 hashed password + """ + self.str = str + + def set(self, value): + self.str = _hashfn(value).hexdigest() + + def __str__(self): + return str(self.str) + + def __eq__(self, other): + if other == None: + return False + return str(_hashfn(other).hexdigest()) == str(self.str) + + def __len__(self): + if self.str: + return len(self.str) + else: + return 0 + +def notify(subject, body=None, html_body=None, to_string=None, attachments=[], append_instance_id=True): + if append_instance_id: + subject = "[%s] %s" % (boto.config.get_value("Instance", "instance-id"), subject) + if not to_string: + to_string = boto.config.get_value('Notification', 'smtp_to', None) + if to_string: + try: + from_string = boto.config.get_value('Notification', 'smtp_from', 'boto') + msg = MIMEMultipart() + msg['From'] = from_string + msg['Reply-To'] = from_string + msg['To'] = to_string + msg['Date'] = formatdate(localtime=True) + msg['Subject'] = subject + + if body: + msg.attach(MIMEText(body)) + + if html_body: + part = MIMEBase('text', 'html') + part.set_payload(html_body) + Encoders.encode_base64(part) + msg.attach(part) + + for part in attachments: + msg.attach(part) + + smtp_host = boto.config.get_value('Notification', 'smtp_host', 'localhost') + + # Alternate port support + if boto.config.get_value("Notification", "smtp_port"): + server = smtplib.SMTP(smtp_host, int(boto.config.get_value("Notification", "smtp_port"))) + else: + server = smtplib.SMTP(smtp_host) + + # TLS support + if boto.config.getbool("Notification", "smtp_tls"): + server.ehlo() + server.starttls() + server.ehlo() + smtp_user = boto.config.get_value('Notification', 'smtp_user', '') + smtp_pass = boto.config.get_value('Notification', 'smtp_pass', '') + if smtp_user: + server.login(smtp_user, smtp_pass) + server.sendmail(from_string, to_string, msg.as_string()) + server.quit() + except: + boto.log.exception('notify failed') + +def get_utf8_value(value): + if not isinstance(value, str) and not isinstance(value, unicode): + value = str(value) + if isinstance(value, unicode): + return value.encode('utf-8') + else: + return value + +def mklist(value): + if not isinstance(value, list): + if isinstance(value, tuple): + value = list(value) + else: + value = [value] + return value + +def pythonize_name(name, sep='_'): + s = '' + if name[0].isupper: + s = name[0].lower() + for c in name[1:]: + if c.isupper(): + s += sep + c.lower() + else: + s += c + return s + +def awsify_name(name): + return name[0:1].upper()+name[1:] diff --git a/backup/src/boto/vpc/__init__.py b/backup/src/boto/vpc/__init__.py new file mode 100644 index 0000000..76eea82 --- /dev/null +++ b/backup/src/boto/vpc/__init__.py @@ -0,0 +1,473 @@ +# Copyright (c) 2009 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a connection to the EC2 service. +""" + +from boto.ec2.connection import EC2Connection +from boto.vpc.vpc import VPC +from boto.vpc.customergateway import CustomerGateway +from boto.vpc.vpngateway import VpnGateway, Attachment +from boto.vpc.dhcpoptions import DhcpOptions +from boto.vpc.subnet import Subnet +from boto.vpc.vpnconnection import VpnConnection + +class VPCConnection(EC2Connection): + + # VPC methods + + def get_all_vpcs(self, vpc_ids=None, filters=None): + """ + Retrieve information about your VPCs. You can filter results to + return information only about those VPCs that match your search + parameters. Otherwise, all VPCs associated with your account + are returned. + + :type vpc_ids: list + :param vpc_ids: A list of strings with the desired VPC ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPC (pending or available) + - *cidrBlock*, CIDR block of the VPC + - *dhcpOptionsId*, the ID of a set of DHCP options + + :rtype: list + :return: A list of :class:`boto.vpc.vpc.VPC` + """ + params = {} + if vpc_ids: + self.build_list_params(params, vpc_ids, 'VpcId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpcs', params, [('item', VPC)]) + + def create_vpc(self, cidr_block): + """ + Create a new Virtual Private Cloud. + + :type cidr_block: str + :param cidr_block: A valid CIDR block + + :rtype: The newly created VPC + :return: A :class:`boto.vpc.vpc.VPC` object + """ + params = {'CidrBlock' : cidr_block} + return self.get_object('CreateVpc', params, VPC) + + def delete_vpc(self, vpc_id): + """ + Delete a Virtual Private Cloud. + + :type vpc_id: str + :param vpc_id: The ID of the vpc to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpcId': vpc_id} + return self.get_status('DeleteVpc', params) + + # Customer Gateways + + def get_all_customer_gateways(self, customer_gateway_ids=None, filters=None): + """ + Retrieve information about your CustomerGateways. You can filter results to + return information only about those CustomerGateways that match your search + parameters. Otherwise, all CustomerGateways associated with your account + are returned. + + :type customer_gateway_ids: list + :param customer_gateway_ids: A list of strings with the desired CustomerGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the CustomerGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *ipAddress* the IP address of customer gateway's + internet-routable external inteface + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.CustomerGateway` + """ + params = {} + if customer_gateway_ids: + self.build_list_params(params, customer_gateway_ids, 'CustomerGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeCustomerGateways', params, [('item', CustomerGateway)]) + + def create_customer_gateway(self, type, ip_address, bgp_asn): + """ + Create a new Customer Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type ip_address: str + :param ip_address: Internet-routable IP address for customer's gateway. + Must be a static address. + + :type bgp_asn: str + :param bgp_asn: Customer gateway's Border Gateway Protocol (BGP) + Autonomous System Number (ASN) + + :rtype: The newly created CustomerGateway + :return: A :class:`boto.vpc.customergateway.CustomerGateway` object + """ + params = {'Type' : type, + 'IpAddress' : ip_address, + 'BgpAsn' : bgp_asn} + return self.get_object('CreateCustomerGateway', params, CustomerGateway) + + def delete_customer_gateway(self, customer_gateway_id): + """ + Delete a Customer Gateway. + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'CustomerGatewayId': customer_gateway_id} + return self.get_status('DeleteCustomerGateway', params) + + # VPN Gateways + + def get_all_vpn_gateways(self, vpn_gateway_ids=None, filters=None): + """ + Retrieve information about your VpnGateways. You can filter results to + return information only about those VpnGateways that match your search + parameters. Otherwise, all VpnGateways associated with your account + are returned. + + :type vpn_gateway_ids: list + :param vpn_gateway_ids: A list of strings with the desired VpnGateway ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VpnGateway + (pending,available,deleting,deleted) + - *type*, the type of customer gateway (ipsec.1) + - *availabilityZone*, the Availability zone the + VPN gateway is in. + + :rtype: list + :return: A list of :class:`boto.vpc.customergateway.VpnGateway` + """ + params = {} + if vpn_gateway_ids: + self.build_list_params(params, vpn_gateway_ids, 'VpnGatewayId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnGateways', params, [('item', VpnGateway)]) + + def create_vpn_gateway(self, type, availability_zone=None): + """ + Create a new Vpn Gateway + + :type type: str + :param type: Type of VPN Connection. Only valid valid currently is 'ipsec.1' + + :type availability_zone: str + :param availability_zone: The Availability Zone where you want the VPN gateway. + + :rtype: The newly created VpnGateway + :return: A :class:`boto.vpc.vpngateway.VpnGateway` object + """ + params = {'Type' : type} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateVpnGateway', params, VpnGateway) + + def delete_vpn_gateway(self, vpn_gateway_id): + """ + Delete a Vpn Gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnGatewayId': vpn_gateway_id} + return self.get_status('DeleteVpnGateway', params) + + def attach_vpn_gateway(self, vpn_gateway_id, vpc_id): + """ + Attaches a VPN gateway to a VPC. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the vpn_gateway to attach + + :type vpc_id: str + :param vpc_id: The ID of the VPC you want to attach the gateway to. + + :rtype: An attachment + :return: a :class:`boto.vpc.vpngateway.Attachment` + """ + params = {'VpnGatewayId': vpn_gateway_id, + 'VpcId' : vpc_id} + return self.get_object('AttachVpnGateway', params, Attachment) + + # Subnets + + def get_all_subnets(self, subnet_ids=None, filters=None): + """ + Retrieve information about your Subnets. You can filter results to + return information only about those Subnets that match your search + parameters. Otherwise, all Subnets associated with your account + are returned. + + :type subnet_ids: list + :param subnet_ids: A list of strings with the desired Subnet ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the Subnet + (pending,available) + - *vpdId*, the ID of teh VPC the subnet is in. + - *cidrBlock*, CIDR block of the subnet + - *availabilityZone*, the Availability Zone + the subnet is in. + + + :rtype: list + :return: A list of :class:`boto.vpc.subnet.Subnet` + """ + params = {} + if subnet_ids: + self.build_list_params(params, subnet_ids, 'SubnetId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1' % i)] = filter[1] + i += 1 + return self.get_list('DescribeSubnets', params, [('item', Subnet)]) + + def create_subnet(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new Subnet + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created Subnet + :return: A :class:`boto.vpc.customergateway.Subnet` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateSubnet', params, Subnet) + + def delete_subnet(self, subnet_id): + """ + Delete a subnet. + + :type subnet_id: str + :param subnet_id: The ID of the subnet to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'SubnetId': subnet_id} + return self.get_status('DeleteSubnet', params) + + + # DHCP Options + + def get_all_dhcp_options(self, dhcp_options_ids=None): + """ + Retrieve information about your DhcpOptions. + + :type dhcp_options_ids: list + :param dhcp_options_ids: A list of strings with the desired DhcpOption ID's + + :rtype: list + :return: A list of :class:`boto.vpc.dhcpoptions.DhcpOptions` + """ + params = {} + if dhcp_options_ids: + self.build_list_params(params, dhcp_options_ids, 'DhcpOptionsId') + return self.get_list('DescribeDhcpOptions', params, [('item', DhcpOptions)]) + + def create_dhcp_options(self, vpc_id, cidr_block, availability_zone=None): + """ + Create a new DhcpOption + + :type vpc_id: str + :param vpc_id: The ID of the VPC where you want to create the subnet. + + :type cidr_block: str + :param cidr_block: The CIDR block you want the subnet to cover. + + :type availability_zone: str + :param availability_zone: The AZ you want the subnet in + + :rtype: The newly created DhcpOption + :return: A :class:`boto.vpc.customergateway.DhcpOption` object + """ + params = {'VpcId' : vpc_id, + 'CidrBlock' : cidr_block} + if availability_zone: + params['AvailabilityZone'] = availability_zone + return self.get_object('CreateDhcpOption', params, DhcpOptions) + + def delete_dhcp_options(self, dhcp_options_id): + """ + Delete a DHCP Options + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the DHCP Options to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': dhcp_options_id} + return self.get_status('DeleteDhcpOptions', params) + + def associate_dhcp_options(self, dhcp_options_id, vpc_id): + """ + Associate a set of Dhcp Options with a VPC. + + :type dhcp_options_id: str + :param dhcp_options_id: The ID of the Dhcp Options + + :type vpc_id: str + :param vpc_id: The ID of the VPC. + + :rtype: bool + :return: True if successful + """ + params = {'DhcpOptionsId': dhcp_options_id, + 'VpcId' : vpc_id} + return self.get_status('AssociateDhcpOptions', params) + + # VPN Connection + + def get_all_vpn_connections(self, vpn_connection_ids=None, filters=None): + """ + Retrieve information about your VPN_CONNECTIONs. You can filter results to + return information only about those VPN_CONNECTIONs that match your search + parameters. Otherwise, all VPN_CONNECTIONs associated with your account + are returned. + + :type vpn_connection_ids: list + :param vpn_connection_ids: A list of strings with the desired VPN_CONNECTION ID's + + :type filters: list of tuples + :param filters: A list of tuples containing filters. Each tuple + consists of a filter key and a filter value. + Possible filter keys are: + + - *state*, the state of the VPN_CONNECTION + pending,available,deleting,deleted + - *type*, the type of connection, currently 'ipsec.1' + - *customerGatewayId*, the ID of the customer gateway + associated with the VPN + - *vpnGatewayId*, the ID of the VPN gateway associated + with the VPN connection + + :rtype: list + :return: A list of :class:`boto.vpn_connection.vpnconnection.VpnConnection` + """ + params = {} + if vpn_connection_ids: + self.build_list_params(params, vpn_connection_ids, 'Vpn_ConnectionId') + if filters: + i = 1 + for filter in filters: + params[('Filter.%d.Key' % i)] = filter[0] + params[('Filter.%d.Value.1')] = filter[1] + i += 1 + return self.get_list('DescribeVpnConnections', params, [('item', VpnConnection)]) + + def create_vpn_connection(self, type, customer_gateway_id, vpn_gateway_id): + """ + Create a new VPN Connection. + + :type type: str + :param type: The type of VPN Connection. Currently only 'ipsec.1' + is supported + + :type customer_gateway_id: str + :param customer_gateway_id: The ID of the customer gateway. + + :type vpn_gateway_id: str + :param vpn_gateway_id: The ID of the VPN gateway. + + :rtype: The newly created VpnConnection + :return: A :class:`boto.vpc.vpnconnection.VpnConnection` object + """ + params = {'Type' : type, + 'CustomerGatewayId' : customer_gateway_id, + 'VpnGatewayId' : vpn_gateway_id} + return self.get_object('CreateVpnConnection', params, VpnConnection) + + def delete_vpn_connection(self, vpn_connection_id): + """ + Delete a VPN Connection. + + :type vpn_connection_id: str + :param vpn_connection_id: The ID of the vpn_connection to be deleted. + + :rtype: bool + :return: True if successful + """ + params = {'VpnConnectionId': vpn_connection_id} + return self.get_status('DeleteVpnConnection', params) + + diff --git a/backup/src/boto/vpc/customergateway.py b/backup/src/boto/vpc/customergateway.py new file mode 100644 index 0000000..959d01f --- /dev/null +++ b/backup/src/boto/vpc/customergateway.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Customer Gateway +""" + +from boto.ec2.ec2object import TaggedEC2Object + +class CustomerGateway(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.ip_address = None + self.bgp_asn = None + + def __repr__(self): + return 'CustomerGateway:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'customerGatewayId': + self.id = value + elif name == 'ipAddress': + self.ip_address = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'bgpAsn': + self.bgp_asn = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/vpc/dhcpoptions.py b/backup/src/boto/vpc/dhcpoptions.py new file mode 100644 index 0000000..810d9cf --- /dev/null +++ b/backup/src/boto/vpc/dhcpoptions.py @@ -0,0 +1,72 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a DHCP Options set +""" + +from boto.ec2.ec2object import TaggedEC2Object + +class DhcpValueSet(list): + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'value': + self.append(value) + +class DhcpConfigSet(dict): + + def startElement(self, name, attrs, connection): + if name == 'valueSet': + if not self.has_key(self._name): + self[self._name] = DhcpValueSet() + return self[self._name] + + def endElement(self, name, value, connection): + if name == 'key': + self._name = value + +class DhcpOptions(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.options = None + + def __repr__(self): + return 'DhcpOptions:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'dhcpConfigurationSet': + self.options = DhcpConfigSet() + return self.options + + def endElement(self, name, value, connection): + if name == 'dhcpOptionsId': + self.id = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/vpc/subnet.py b/backup/src/boto/vpc/subnet.py new file mode 100644 index 0000000..135e1a2 --- /dev/null +++ b/backup/src/boto/vpc/subnet.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Subnet +""" + +from boto.ec2.ec2object import TaggedEC2Object + +class Subnet(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.state = None + self.cidr_block = None + self.available_ip_address_count = 0 + self.availability_zone = None + + def __repr__(self): + return 'Subnet:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'subnetId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + elif name == 'availableIpAddressCount': + self.available_ip_address_count = int(value) + elif name == 'availabilityZone': + self.availability_zone = value + else: + setattr(self, name, value) + diff --git a/backup/src/boto/vpc/vpc.py b/backup/src/boto/vpc/vpc.py new file mode 100644 index 0000000..0539acd --- /dev/null +++ b/backup/src/boto/vpc/vpc.py @@ -0,0 +1,54 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Virtual Private Cloud. +""" + +from boto.ec2.ec2object import TaggedEC2Object + +class VPC(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.dhcp_options_id = None + self.state = None + self.cidr_block = None + + def __repr__(self): + return 'VPC:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.id = value + elif name == 'dhcpOptionsId': + self.dhcp_options_id = value + elif name == 'state': + self.state = value + elif name == 'cidrBlock': + self.cidr_block = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpc(self.id) + diff --git a/backup/src/boto/vpc/vpnconnection.py b/backup/src/boto/vpc/vpnconnection.py new file mode 100644 index 0000000..2e089e7 --- /dev/null +++ b/backup/src/boto/vpc/vpnconnection.py @@ -0,0 +1,60 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a VPN Connectionn +""" + +from boto.ec2.ec2object import EC2Object + +class VpnConnection(EC2Object): + + def __init__(self, connection=None): + EC2Object.__init__(self, connection) + self.id = None + self.state = None + self.customer_gateway_configuration = None + self.type = None + self.customer_gateway_id = None + self.vpn_gateway_id = None + + def __repr__(self): + return 'VpnConnection:%s' % self.id + + def endElement(self, name, value, connection): + if name == 'vpnConnectionId': + self.id = value + elif name == 'state': + self.state = value + elif name == 'CustomerGatewayConfiguration': + self.customer_gateway_configuration = value + elif name == 'type': + self.type = value + elif name == 'customerGatewayId': + self.customer_gateway_id = value + elif name == 'vpnGatewayId': + self.vpn_gateway_id = value + else: + setattr(self, name, value) + + def delete(self): + return self.connection.delete_vpn_connection(self.id) + diff --git a/backup/src/boto/vpc/vpngateway.py b/backup/src/boto/vpc/vpngateway.py new file mode 100644 index 0000000..83b912e --- /dev/null +++ b/backup/src/boto/vpc/vpngateway.py @@ -0,0 +1,83 @@ +# Copyright (c) 2009-2010 Mitch Garnaat http://garnaat.org/ +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + +""" +Represents a Vpn Gateway +""" + +from boto.ec2.ec2object import TaggedEC2Object + +class Attachment(object): + + def __init__(self, connection=None): + self.vpc_id = None + self.state = None + + def startElement(self, name, attrs, connection): + pass + + def endElement(self, name, value, connection): + if name == 'vpcId': + self.vpc_id = value + elif name == 'state': + self.state = value + else: + setattr(self, name, value) + +class VpnGateway(TaggedEC2Object): + + def __init__(self, connection=None): + TaggedEC2Object.__init__(self, connection) + self.id = None + self.type = None + self.state = None + self.availability_zone = None + self.attachments = [] + + def __repr__(self): + return 'VpnGateway:%s' % self.id + + def startElement(self, name, attrs, connection): + retval = TaggedEC2Object.startElement(self, name, attrs, connection) + if retval is not None: + return retval + if name == 'item': + att = Attachment() + self.attachments.append(att) + return att + + def endElement(self, name, value, connection): + if name == 'vpnGatewayId': + self.id = value + elif name == 'type': + self.type = value + elif name == 'state': + self.state = value + elif name == 'availabilityZone': + self.availability_zone = value + elif name == 'attachments': + pass + else: + setattr(self, name, value) + + def attach(self, vpc_id): + return self.connection.attach_vpn_gateway(self.id, vpc_id) +