diff --git a/scripts/cloudinary/.gitignore b/scripts/cloudinary/.gitignore new file mode 100644 index 0000000..567609b --- /dev/null +++ b/scripts/cloudinary/.gitignore @@ -0,0 +1 @@ +build/ diff --git a/scripts/cloudinary/__init__.py b/scripts/cloudinary/__init__.py new file mode 100644 index 0000000..d483c65 --- /dev/null +++ b/scripts/cloudinary/__init__.py @@ -0,0 +1,125 @@ +from __future__ import absolute_import + +import os +import sys + +CF_SHARED_CDN = "d3jpl91pxevbkh.cloudfront.net" +OLD_AKAMAI_SHARED_CDN = "cloudinary-a.akamaihd.net" +AKAMAI_SHARED_CDN = "res.cloudinary.com" +SHARED_CDN = AKAMAI_SHARED_CDN +CL_BLANK = "data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7" + +VERSION = "1.0.21" +USER_AGENT = "cld-python-" + VERSION + +from cloudinary import utils +from cloudinary.compat import urlparse, parse_qs + +def import_django_settings(): + try: + import django.conf + from django.core.exceptions import ImproperlyConfigured + try: + if 'CLOUDINARY' in dir(django.conf.settings): + return django.conf.settings.CLOUDINARY + else: + return None + except ImproperlyConfigured: + return None + except ImportError: + return None + +class Config(object): + def __init__(self): + django_settings = import_django_settings() + if django_settings: + self.update(**django_settings) + elif os.environ.get("CLOUDINARY_CLOUD_NAME"): + self.update( + cloud_name = os.environ.get("CLOUDINARY_CLOUD_NAME"), + api_key = os.environ.get("CLOUDINARY_API_KEY"), + api_secret = os.environ.get("CLOUDINARY_API_SECRET"), + secure_distribution = os.environ.get("CLOUDINARY_SECURE_DISTRIBUTION"), + private_cdn = os.environ.get("CLOUDINARY_PRIVATE_CDN") == 'true' + ) + elif os.environ.get("CLOUDINARY_URL"): + uri = urlparse(os.environ.get("CLOUDINARY_URL").replace("cloudinary://", "http://")) + for k, v in parse_qs(uri.query).items(): + self.__dict__[k] = v[0] + self.update( + cloud_name = uri.hostname, + api_key = uri.username, + api_secret = uri.password, + private_cdn = uri.path != '' + ) + if uri.path != '': + self.update(secure_distribution = uri.path[1:]) + def __getattr__(self, i): + if i in self.__dict__: + return self.__dict__[i] + else: + return None + + def update(self, **keywords): + for k, v in keywords.items(): + self.__dict__[k] = v + +_config = Config() + +def config(**keywords): + global _config + _config.update(**keywords) + return _config + +def reset_config(): + global _config + _config = Config() + +class CloudinaryImage(object): + def __init__(self, public_id = None, format = None, version = None, + signature = None, url_options = {}, metadata = None, type = None): + + self.metadata = metadata + metadata = metadata or {} + self.public_id = public_id or metadata.get('public_id') + self.format = format or metadata.get('format') + self.version = version or metadata.get('version') + self.signature = signature or metadata.get('signature') + self.type = type or metadata.get('type') or "upload" + self.url_options = url_options + + def __unicode__(self): + return self.public_id + + def validate(self): + expected = utils.api_sign_request({"public_id": self.public_id, "version": self.version}, config().api_secret) + return self.signature == expected + + @property + def url(self): + return self.build_url(**self.url_options) + + def __build_url(self, **options): + combined_options = dict(format = self.format, version = self.version, type = self.type) + combined_options.update(options) + return utils.cloudinary_url(self.public_id, **combined_options) + + def build_url(self, **options): + return self.__build_url(**options)[0] + + def image(self, **options): + src, attrs = self.__build_url(**options) + responsive = attrs.pop("responsive", False) + hidpi = attrs.pop("hidpi", False) + if responsive or hidpi: + attrs["data-src"] = src + classes = "cld-responsive" if responsive else "cld-hidpi" + if "class" in attrs: classes += " " + attrs["class"] + attrs["class"] = classes + src = attrs.pop("responsive_placeholder", config().responsive_placeholder) + if src == "blank": src = CL_BLANK + + attrs = sorted(attrs.items()) + if src: attrs.insert(0, ("src", src)) + + return u"".format(' '.join([u"{0}='{1}'".format(key, value) for key, value in attrs if value])) diff --git a/scripts/cloudinary/__init__.pyc b/scripts/cloudinary/__init__.pyc new file mode 100644 index 0000000..f50d39f Binary files /dev/null and b/scripts/cloudinary/__init__.pyc differ diff --git a/scripts/cloudinary/api.py b/scripts/cloudinary/api.py new file mode 100644 index 0000000..5e75356 --- /dev/null +++ b/scripts/cloudinary/api.py @@ -0,0 +1,243 @@ +# Copyright Cloudinary +import json +import base64 +import sys +import email.utils +import socket +import cloudinary +from cloudinary.compat import urllib2, urlencode, to_string, to_bytes, PY3, HTTPError +from cloudinary import utils + +class Error(Exception): pass +class NotFound(Error): pass +class NotAllowed(Error): pass +class AlreadyExists(Error): pass +class RateLimited(Error): pass +class BadRequest(Error): pass +class GeneralError(Error): pass +class AuthorizationRequired(Error): pass + +EXCEPTION_CODES = { + 400: BadRequest, + 401: AuthorizationRequired, + 403: NotAllowed, + 404: NotFound, + 409: AlreadyExists, + 420: RateLimited +} + +class Response(dict): + def __init__(self, result, response): + self.update(result) + self.rate_limit_allowed = int(response.headers["x-featureratelimit-limit"]) + self.rate_limit_reset_at = email.utils.parsedate(response.headers["x-featureratelimit-reset"]) + self.rate_limit_remaining = int(response.headers["x-featureratelimit-remaining"]) + +def ping(**options): + return call_api("get", ["ping"], {}, **options) + +def usage(**options): + return call_api("get", ["usage"], {}, **options) + +def resource_types(**options): + return call_api("get", ["resources"], {}, **options) + +def resources(**options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", None) + uri = ["resources", resource_type] + if type: uri.append(type) + return call_api("get", uri, only(options, "next_cursor", "max_results", "prefix", "tags", "context", "moderations", "direction", "start_at"), **options) + +def resources_by_tag(tag, **options): + resource_type = options.pop("resource_type", "image") + uri = ["resources", resource_type, "tags", tag] + return call_api("get", uri, only(options, "next_cursor", "max_results", "tags", "context", "moderations", "direction"), **options) + +def resources_by_moderation(kind, status, **options): + resource_type = options.pop("resource_type", "image") + uri = ["resources", resource_type, "moderations", kind, status] + return call_api("get", uri, only(options, "next_cursor", "max_results", "tags", "context", "moderations", "direction"), **options) + +def resources_by_ids(public_ids, **options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type] + params = [("public_ids[]", public_id) for public_id in public_ids] + optional = list(only(options, "tags", "moderations", "context").items()) + return call_api("get", uri, params + optional, **options) + +def resource(public_id, **options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type, public_id] + return call_api("get", uri, only(options, "exif", "faces", "colors", "image_metadata", "pages", "phash", "coordinates", "max_results"), **options) + +def update(public_id, **options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type, public_id] + upload_options = only(options, "moderation_status", "raw_convert", "ocr", "categorization", "detection", "similarity_search", "background_removal") + if "tags" in options: upload_options["tags"] = ",".join(utils.build_array(options["tags"])) + if "face_coordinates" in options: upload_options["face_coordinates"] = utils.encode_double_array(options.get("face_coordinates")) + if "custom_coordinates" in options: upload_options["custom_coordinates"] = utils.encode_double_array(options.get("custom_coordinates")) + if "context" in options: upload_options["context"] = utils.encode_dict(options.get("context")) + if "auto_tagging" in options: upload_options["auto_tagging"] = float(options.get("auto_tagging")) + return call_api("post", uri, upload_options, **options) + +def delete_resources(public_ids, **options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type] + params = [("public_ids[]", public_id) for public_id in public_ids] + optional = list(only(options, "keep_original", "next_cursor", "invalidate").items()) + return call_api("delete", uri, params + optional, **options) + +def delete_resources_by_prefix(prefix, **options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type] + return call_api("delete", uri, dict(only(options, "keep_original", "next_cursor", "invalidate"), prefix=prefix), **options) + +def delete_all_resources(**options): + resource_type = options.pop("resource_type", "image") + type = options.pop("type", "upload") + uri = ["resources", resource_type, type] + optional = list(only(options, "keep_original", "next_cursor").items()) + return call_api("delete", uri, dict(only(options, "keep_original", "next_cursor", "invalidate"), all=True), **options) + +def delete_resources_by_tag(tag, **options): + resource_type = options.pop("resource_type", "image") + uri = ["resources", resource_type, "tags", tag] + return call_api("delete", uri, only(options, "keep_original", "next_cursor", "invalidate"), **options) + +def delete_derived_resources(derived_resource_ids, **options): + uri = ["derived_resources"] + params = [("derived_resource_ids[]", derived_resource_id) for derived_resource_id in derived_resource_ids] + + return call_api("delete", uri, params, **options) + +def tags(**options): + resource_type = options.pop("resource_type", "image") + uri = ["tags", resource_type] + return call_api("get", uri, only(options, "next_cursor", "max_results", "prefix"), **options) + +def transformations(**options): + uri = ["transformations"] + return call_api("get", uri, only(options, "next_cursor", "max_results"), **options) + +def transformation(transformation, **options): + uri = ["transformations", transformation_string(transformation)] + return call_api("get", uri, only(options, "max_results"), **options) + +def delete_transformation(transformation, **options): + uri = ["transformations", transformation_string(transformation)] + return call_api("delete", uri, {}, **options) + +# updates - currently only supported update is the "allowed_for_strict" boolean flag and unsafe_update +def update_transformation(transformation, **options): + uri = ["transformations", transformation_string(transformation)] + updates = only(options, "allowed_for_strict") + if "unsafe_update" in options: + updates["unsafe_update"] = transformation_string(options.get("unsafe_update")) + if len(updates) == 0: raise Exception("No updates given") + + return call_api("put", uri, updates, **options) + +def create_transformation(name, definition, **options): + uri = ["transformations", name] + return call_api("post", uri, {"transformation": transformation_string(definition)}, **options) + +def upload_presets(**options): + uri = ["upload_presets"] + return call_api("get", uri, only(options, "next_cursor", "max_results"), **options) + +def upload_preset(name, **options): + uri = ["upload_presets", name] + return call_api("get", uri, only(options, "max_results"), **options) + +def delete_upload_preset(name, **options): + uri = ["upload_presets", name] + return call_api("delete", uri, {}, **options) + +def update_upload_preset(name, **options): + uri = ["upload_presets", name] + params = utils.build_upload_params(**options) + params = utils.cleanup_params(params) + params.update(only(options, "unsigned", "disallow_public_id")) + return call_api("put", uri, params, **options) + +def create_upload_preset(**options): + uri = ["upload_presets"] + params = utils.build_upload_params(**options) + params = utils.cleanup_params(params) + params.update(only(options, "unsigned", "disallow_public_id", "name")) + return call_api("post", uri, params, **options) + +def root_folders(**options): + return call_api("get", ["folders"], {}, **options) + +def subfolders(of_folder_path, **options): + return call_api("get", ["folders", of_folder_path], {}, **options) + +def call_api(method, uri, params, **options): + prefix = options.pop("upload_prefix", cloudinary.config().upload_prefix) or "https://api.cloudinary.com" + cloud_name = options.pop("cloud_name", cloudinary.config().cloud_name) + if not cloud_name: raise Exception("Must supply cloud_name") + api_key = options.pop("api_key", cloudinary.config().api_key) + if not api_key: raise Exception("Must supply api_key") + api_secret = options.pop("api_secret", cloudinary.config().api_secret) + if not cloud_name: raise Exception("Must supply api_secret") + + data = to_bytes(urlencode(params)) + api_url = "/".join([prefix, "v1_1", cloud_name] + uri) + request = urllib2.Request(api_url, data) + # Add authentication + byte_value = to_bytes('%s:%s' % (api_key, api_secret)) + encoded_value = base64.encodebytes(byte_value) if PY3 else base64.encodestring(byte_value) + base64string = to_string(encoded_value).replace('\n', '') + request.add_header("Authorization", "Basic %s" % base64string) + request.add_header("User-Agent", cloudinary.USER_AGENT) + request.get_method = lambda: method.upper() + + kw = {} + if 'timeout' in options: + kw['timeout'] = options['timeout'] + try: + response = urllib2.urlopen(request, **kw) + body = response.read() + except HTTPError: + e = sys.exc_info()[1] + exception_class = EXCEPTION_CODES.get(e.code) + if exception_class: + response = e + body = response.read() + else: + raise GeneralError("Server returned unexpected status code - %d - %s" % (e.code, e.read())) + except socket.error: + e = sys.exc_info()[1] + raise GeneralError("Socket Error: %s" % (str(e))) + + try: + body = to_string(body) + result = json.loads(body) + except Exception: + # Error is parsing json + e = sys.exc_info()[1] + raise GeneralError("Error parsing server response (%d) - %s. Got - %s" % (response.code, body, e)) + + if "error" in result: + exception_class = exception_class or Exception + raise exception_class(result["error"]["message"]) + + return Response(result, response) + +def only(hash, *keys): + result = {} + for key in keys: + if key in hash: result[key] = hash[key] + return result + +def transformation_string(transformation): + return transformation if isinstance(transformation, str) else cloudinary.utils.generate_transformation_string(**transformation)[0] + diff --git a/scripts/cloudinary/api.pyc b/scripts/cloudinary/api.pyc new file mode 100644 index 0000000..311545a Binary files /dev/null and b/scripts/cloudinary/api.pyc differ diff --git a/scripts/cloudinary/compat.py b/scripts/cloudinary/compat.py new file mode 100644 index 0000000..ff2b749 --- /dev/null +++ b/scripts/cloudinary/compat.py @@ -0,0 +1,36 @@ +# Copyright Cloudinary +import sys + +PY3 = (sys.version_info[0] == 3) + +if PY3: + import http.client as httplib + NotConnected = httplib.NotConnected + import urllib.request as urllib2 + import urllib.error + HTTPError = urllib.error.HTTPError + from io import StringIO, BytesIO + from urllib.parse import urlencode, unquote, urlparse, parse_qs, quote_plus + to_bytes = lambda s: s.encode('utf8') + to_bytearray = lambda s: bytearray(s, 'utf8') + to_string = lambda b: b.decode('utf8') + string_types = (str) +else: + import httplib + from httplib import NotConnected + from io import BytesIO + import StringIO + import urllib2 + HTTPError = urllib2.HTTPError + from urllib import urlencode, unquote, quote_plus + from urlparse import urlparse, parse_qs + to_bytes = str + to_bytearray = str + to_string = str + string_types = (str, unicode) + +try: + advance_iterator = next +except NameError: + def advance_iterator(it): + return it.next() diff --git a/scripts/cloudinary/compat.pyc b/scripts/cloudinary/compat.pyc new file mode 100644 index 0000000..e8ca06c Binary files /dev/null and b/scripts/cloudinary/compat.pyc differ diff --git a/scripts/cloudinary/forms.py b/scripts/cloudinary/forms.py new file mode 100644 index 0000000..9ccb4ea --- /dev/null +++ b/scripts/cloudinary/forms.py @@ -0,0 +1,114 @@ +from django import forms +from cloudinary import CloudinaryImage +import cloudinary.uploader +import cloudinary.utils +import re +import json +from django.utils.translation import ugettext_lazy as _ + +def cl_init_js_callbacks(form, request): + for field in form.fields.values(): + if (isinstance(field, CloudinaryJsFileField)): + field.enable_callback(request) + +class CloudinaryInput(forms.TextInput): + input_type = 'file' + + def render(self, name, value, attrs=None): + attrs = self.build_attrs(attrs) + options = attrs.get('options', {}) + attrs["options"] = '' + + params = cloudinary.utils.build_upload_params(**options) + if options.get("unsigned"): + params = cloudinary.utils.cleanup_params(params) + else: + params = cloudinary.utils.sign_request(params, options) + + if 'resource_type' not in options: options['resource_type'] = 'auto' + cloudinary_upload_url = cloudinary.utils.cloudinary_api_url("upload", **options) + + attrs["data-url"] = cloudinary_upload_url + attrs["data-form-data"] = json.dumps(params) + attrs["data-cloudinary-field"] = name + attrs["class"] = " ".join(["cloudinary-fileupload", attrs.get("class", "")]) + + widget = super(CloudinaryInput, self).render("file", None, attrs=attrs) + if value: widget += forms.HiddenInput().render(name, value) + return widget + + +class CloudinaryJsFileField(forms.Field): + default_error_messages = { + 'required': _(u"No image selected!") + } + + def __init__(self, attrs={}, options={}, autosave=True, *args, **kwargs): + self.autosave = autosave + attrs = attrs.copy() + attrs["options"] = options.copy() + + field_options = {'widget': CloudinaryInput(attrs=attrs)} + field_options.update(kwargs) + super(CloudinaryJsFileField, self).__init__(*args, **field_options) + + def enable_callback(self, request): + from django.contrib.staticfiles.storage import staticfiles_storage + self.widget.attrs["options"]["callback"] = request.build_absolute_uri(staticfiles_storage.url("html/cloudinary_cors.html")) + + def to_python(self, value): + "Convert to CloudinaryImage" + if not value: + return None; + m = re.search(r'^([^/]+)/([^/]+)/v(\d+)/([^#]+)#([^/]+)$', value) + if not m: + raise forms.ValidationError("Invalid format") + resource_type = m.group(1) + if resource_type != 'image': + raise forms.ValidationError("Only images are supported") + image_type = m.group(2) + version = m.group(3) + filename = m.group(4) + signature = m.group(5) + m = re.search(r'(.*)\.(.*)', filename) + if not m: + raise forms.ValidationError("Invalid file name") + public_id = m.group(1) + image_format = m.group(2) + return CloudinaryImage(public_id, format=image_format, version=version, signature=signature, type=image_type) + + def validate(self, value): + "Validate the signature" + # Use the parent's handling of required fields, etc. + super(CloudinaryJsFileField, self).validate(value) + if not value: return + if not value.validate(): + raise forms.ValidationError("Signature mismatch") + +class CloudinaryUnsignedJsFileField(CloudinaryJsFileField): + + def __init__(self, upload_preset, attrs={}, options={}, autosave=True, *args, **kwargs): + options = options.copy() + options.update({"unsigned": True, "upload_preset": upload_preset}) + super(CloudinaryUnsignedJsFileField, self).__init__(attrs, options, autosave, *args, **kwargs) + +class CloudinaryFileField(forms.FileField): + my_default_error_messages = { + 'required': _(u"No image selected!") + } + default_error_messages = forms.FileField.default_error_messages.copy() + default_error_messages.update(my_default_error_messages) + def __init__(self, options=None, autosave=True, *args, **kwargs): + self.autosave = autosave + self.options = options or {} + super(CloudinaryFileField, self).__init__(*args, **kwargs) + + def to_python(self, value): + "Upload and convert to CloudinaryImage" + value = super(CloudinaryFileField, self).to_python(value) + if not value: + return None; + if self.autosave: + return cloudinary.uploader.upload_image(value, **self.options) + else: + return value diff --git a/scripts/cloudinary/models.py b/scripts/cloudinary/models.py new file mode 100644 index 0000000..6461826 --- /dev/null +++ b/scripts/cloudinary/models.py @@ -0,0 +1,78 @@ +import re +from cloudinary import CloudinaryImage, forms, uploader +from django.db import models +from django.core.files.uploadedfile import UploadedFile + +# Add introspection rules for South, if it's installed. +try: + from south.modelsinspector import add_introspection_rules + add_introspection_rules([], ["^cloudinary.models.CloudinaryField"]) +except ImportError: + pass + +class CloudinaryField(models.Field): + + description = "An image stored in Cloudinary" + + __metaclass__ = models.SubfieldBase + + def __init__(self, *args, **kwargs): + options = {'max_length': 100} + self.default_form_class = kwargs.pop("default_form_class", forms.CloudinaryFileField) + options.update(kwargs) + self.type = options.pop("type", "upload") + super(CloudinaryField, self).__init__(*args, **options) + + def get_internal_type(self): + return 'CharField' + + def value_to_string(self, obj): + value = self._get_val_from_obj(obj) + return self.get_prep_value(value) + + def to_python(self, value): + if isinstance(value, CloudinaryImage): + return value + elif isinstance(value, UploadedFile): + return value + elif not value: + return value + else: + m = re.search('(v(?P\d+)/)?(?P.*?)(\.(?P[^.]+))?$', value) + return CloudinaryImage(type = self.type, **m.groupdict()) + + def upload_options_with_filename(self, model_instance, filename): + return self.upload_options(model_instance); + + def upload_options(self, model_instance): + return {} + + def pre_save(self, model_instance, add): + value = super(CloudinaryField, self).pre_save(model_instance, add) + if isinstance(value, UploadedFile): + options = {"type": self.type} + options.update(self.upload_options_with_filename(model_instance, value.name)) + instance_value = uploader.upload_image(value, **options) + setattr(model_instance, self.attname, instance_value) + return self.get_prep_value(instance_value) + else: + return value + + def get_prep_value(self, value): + prep = '' + if not value: + return None + if isinstance(value, CloudinaryImage): + if value.version: prep = prep + 'v' + str(value.version) + '/' + prep = prep + value.public_id + if value.format: prep = prep + '.' + value.format + return prep + else: + return value + + def formfield(self, **kwargs): + options = {"type": "upload"} + options.update(kwargs.pop('options', {})) + defaults = {'form_class': self.default_form_class, 'options': options} + defaults.update(kwargs) + return super(CloudinaryField, self).formfield(autosave=False, **defaults) diff --git a/scripts/cloudinary/poster/__init__.py b/scripts/cloudinary/poster/__init__.py new file mode 100644 index 0000000..9110fa4 --- /dev/null +++ b/scripts/cloudinary/poster/__init__.py @@ -0,0 +1,34 @@ +# MIT licensed code copied from https://bitbucket.org/chrisatlee/poster +# +# Copyright (c) 2011 Chris AtLee +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +# THE SOFTWARE. +"""poster module + +Support for streaming HTTP uploads, and multipart/form-data encoding + +```poster.version``` is a 3-tuple of integers representing the version number. +New releases of poster will always have a version number that compares greater +than an older version of poster. +New in version 0.6.""" + +import cloudinary.poster.streaminghttp +import cloudinary.poster.encode + +version = (0, 8, 2) # Thanks JP! diff --git a/scripts/cloudinary/poster/__init__.pyc b/scripts/cloudinary/poster/__init__.pyc new file mode 100644 index 0000000..5450364 Binary files /dev/null and b/scripts/cloudinary/poster/__init__.pyc differ diff --git a/scripts/cloudinary/poster/encode.py b/scripts/cloudinary/poster/encode.py new file mode 100644 index 0000000..0104cd6 --- /dev/null +++ b/scripts/cloudinary/poster/encode.py @@ -0,0 +1,441 @@ +# MIT licensed code copied from https://bitbucket.org/chrisatlee/poster +"""multipart/form-data encoding module + +This module provides functions that faciliate encoding name/value pairs +as multipart/form-data suitable for a HTTP POST or PUT request. + +multipart/form-data is the standard way to upload files over HTTP""" + +__all__ = ['gen_boundary', 'encode_and_quote', 'MultipartParam', + 'encode_string', 'encode_file_header', 'get_body_size', 'get_headers', + 'multipart_encode'] + +try: + from io import UnsupportedOperation +except ImportError: + UnsupportedOperation = None + +try: + import uuid + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + return uuid.uuid4().hex +except ImportError: + import random, sha + def gen_boundary(): + """Returns a random string to use as the boundary for a message""" + bits = random.getrandbits(160) + return sha.new(str(bits)).hexdigest() + +import re, os, mimetypes +from cloudinary.compat import (PY3, string_types, to_bytes, to_string, + to_bytearray, quote_plus, advance_iterator) +try: + from email.header import Header +except ImportError: + # Python 2.4 + from email.Header import Header + +if PY3: + def encode_and_quote(data): + if data is None: + return None + return quote_plus(to_bytes(data)) + +else: + def encode_and_quote(data): + """If ``data`` is unicode, return quote_plus(data.encode("utf-8")) otherwise return quote_plus(data)""" + if data is None: + return None + + if isinstance(data, unicode): + data = data.encode("utf-8") + return quote_plus(data) + +if PY3: + def _strify(s): + if s is None: + return None + return to_bytes(s) +else: + def _strify(s): + """If s is a unicode string, encode it to UTF-8 and return the results, otherwise return str(s), or None if s is None""" + if s is None: + return None + if isinstance(s, unicode): + return s.encode("utf-8") + return str(s) + +class MultipartParam(object): + """Represents a single parameter in a multipart/form-data request + + ``name`` is the name of this parameter. + + If ``value`` is set, it must be a string or unicode object to use as the + data for this parameter. + + If ``filename`` is set, it is what to say that this parameter's filename + is. Note that this does not have to be the actual filename any local file. + + If ``filetype`` is set, it is used as the Content-Type for this parameter. + If unset it defaults to "text/plain; charset=utf8" + + If ``filesize`` is set, it specifies the length of the file ``fileobj`` + + If ``fileobj`` is set, it must be a file-like object that supports + .read(). + + Both ``value`` and ``fileobj`` must not be set, doing so will + raise a ValueError assertion. + + If ``fileobj`` is set, and ``filesize`` is not specified, then + the file's size will be determined first by stat'ing ``fileobj``'s + file descriptor, and if that fails, by seeking to the end of the file, + recording the current position as the size, and then by seeking back to the + beginning of the file. + + ``cb`` is a callable which will be called from iter_encode with (self, + current, total), representing the current parameter, current amount + transferred, and the total size. + """ + def __init__(self, name, value=None, filename=None, filetype=None, + filesize=None, fileobj=None, cb=None): + self.name = Header(name).encode() + self.value = _strify(value) + if filename is None: + self.filename = None + else: + if PY3: + byte_filename = filename.encode("ascii", "xmlcharrefreplace") + self.filename = to_string(byte_filename) + encoding = 'unicode_escape' + else: + if isinstance(filename, unicode): + # Encode with XML entities + self.filename = filename.encode("ascii", "xmlcharrefreplace") + else: + self.filename = str(filename) + encoding = 'string_escape' + self.filename = self.filename.encode(encoding).replace(to_bytes('"'), to_bytes('\\"')) + self.filetype = _strify(filetype) + + self.filesize = filesize + self.fileobj = fileobj + self.cb = cb + + if self.value is not None and self.fileobj is not None: + raise ValueError("Only one of value or fileobj may be specified") + + if fileobj is not None and filesize is None: + # Try and determine the file size + try: + self.filesize = os.fstat(fileobj.fileno()).st_size + except (OSError, AttributeError, UnsupportedOperation): + try: + fileobj.seek(0, 2) + self.filesize = fileobj.tell() + fileobj.seek(0) + except: + raise ValueError("Could not determine filesize") + + def __cmp__(self, other): + attrs = ['name', 'value', 'filename', 'filetype', 'filesize', 'fileobj'] + myattrs = [getattr(self, a) for a in attrs] + oattrs = [getattr(other, a) for a in attrs] + return cmp(myattrs, oattrs) + + def reset(self): + if self.fileobj is not None: + self.fileobj.seek(0) + elif self.value is None: + raise ValueError("Don't know how to reset this parameter") + + @classmethod + def from_file(cls, paramname, filename): + """Returns a new MultipartParam object constructed from the local + file at ``filename``. + + ``filesize`` is determined by os.path.getsize(``filename``) + + ``filetype`` is determined by mimetypes.guess_type(``filename``)[0] + + ``filename`` is set to os.path.basename(``filename``) + """ + + return cls(paramname, filename=os.path.basename(filename), + filetype=mimetypes.guess_type(filename)[0], + filesize=os.path.getsize(filename), + fileobj=open(filename, "rb")) + + @classmethod + def from_params(cls, params): + """Returns a list of MultipartParam objects from a sequence of + name, value pairs, MultipartParam instances, + or from a mapping of names to values + + The values may be strings or file objects, or MultipartParam objects. + MultipartParam object names must match the given names in the + name,value pairs or mapping, if applicable.""" + if hasattr(params, 'items'): + params = params.items() + + retval = [] + for item in params: + if isinstance(item, cls): + retval.append(item) + continue + name, value = item + if isinstance(value, cls): + assert value.name == name + retval.append(value) + continue + if hasattr(value, 'read'): + # Looks like a file object + filename = getattr(value, 'name', None) + if filename is not None: + filetype = mimetypes.guess_type(filename)[0] + else: + filetype = None + + retval.append(cls(name=name, filename=filename, + filetype=filetype, fileobj=value)) + else: + retval.append(cls(name, value)) + return retval + + def encode_hdr(self, boundary): + """Returns the header of the encoding of this parameter""" + boundary = encode_and_quote(boundary) + + headers = ["--%s" % boundary] + + if self.filename: + disposition = 'form-data; name="%s"; filename="%s"' % (self.name, + to_string(self.filename)) + else: + disposition = 'form-data; name="%s"' % self.name + + headers.append("Content-Disposition: %s" % disposition) + + if self.filetype: + filetype = to_string(self.filetype) + else: + filetype = "text/plain; charset=utf-8" + + headers.append("Content-Type: %s" % filetype) + + headers.append("") + headers.append("") + + return "\r\n".join(headers) + + def encode(self, boundary): + """Returns the string encoding of this parameter""" + if self.value is None: + value = self.fileobj.read() + else: + value = self.value + + if re.search("^--%s$" % re.escape(boundary), value, re.M): + raise ValueError("boundary found in encoded string") + + return "%s%s\r\n" % (self.encode_hdr(boundary), value) + + def iter_encode(self, boundary, blocksize=4096): + """Yields the encoding of this parameter + If self.fileobj is set, then blocks of ``blocksize`` bytes are read and + yielded.""" + total = self.get_size(boundary) + current = 0 + if self.value is not None: + block = to_bytes(self.encode(boundary)) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + else: + block = to_bytes(self.encode_hdr(boundary)) + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + last_block = to_bytearray("") + encoded_boundary = "--%s" % encode_and_quote(boundary) + boundary_exp = re.compile(to_bytes("^%s$" % re.escape(encoded_boundary)), + re.M) + while True: + block = self.fileobj.read(blocksize) + if not block: + current += 2 + yield to_bytes("\r\n") + if self.cb: + self.cb(self, current, total) + break + last_block += block + if boundary_exp.search(last_block): + raise ValueError("boundary found in file data") + last_block = last_block[-len(to_bytes(encoded_boundary))-2:] + current += len(block) + yield block + if self.cb: + self.cb(self, current, total) + + def get_size(self, boundary): + """Returns the size in bytes that this param will be when encoded + with the given boundary.""" + if self.filesize is not None: + valuesize = self.filesize + else: + valuesize = len(self.value) + + return len(self.encode_hdr(boundary)) + 2 + valuesize + +def encode_string(boundary, name, value): + """Returns ``name`` and ``value`` encoded as a multipart/form-data + variable. ``boundary`` is the boundary string used throughout + a single request to separate variables.""" + + return MultipartParam(name, value).encode(boundary) + +def encode_file_header(boundary, paramname, filesize, filename=None, + filetype=None): + """Returns the leading data for a multipart/form-data field that contains + file data. + + ``boundary`` is the boundary string used throughout a single request to + separate variables. + + ``paramname`` is the name of the variable in this request. + + ``filesize`` is the size of the file data. + + ``filename`` if specified is the filename to give to this field. This + field is only useful to the server for determining the original filename. + + ``filetype`` if specified is the MIME type of this file. + + The actual file data should be sent after this header has been sent. + """ + + return MultipartParam(paramname, filesize=filesize, filename=filename, + filetype=filetype).encode_hdr(boundary) + +def get_body_size(params, boundary): + """Returns the number of bytes that the multipart/form-data encoding + of ``params`` will be.""" + size = sum(p.get_size(boundary) for p in MultipartParam.from_params(params)) + return size + len(boundary) + 6 + +def get_headers(params, boundary): + """Returns a dictionary with Content-Type and Content-Length headers + for the multipart/form-data encoding of ``params``.""" + headers = {} + boundary = quote_plus(boundary) + headers['Content-Type'] = "multipart/form-data; boundary=%s" % boundary + headers['Content-Length'] = str(get_body_size(params, boundary)) + return headers + +class multipart_yielder: + def __init__(self, params, boundary, cb): + self.params = params + self.boundary = boundary + self.cb = cb + + self.i = 0 + self.p = None + self.param_iter = None + self.current = 0 + self.total = get_body_size(params, boundary) + + def __iter__(self): + return self + + def __next__(self): + return self.next() + + def next(self): + """generator function to yield multipart/form-data representation + of parameters""" + if self.param_iter is not None: + try: + block = advance_iterator(self.param_iter) + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + except StopIteration: + self.p = None + self.param_iter = None + + if self.i is None: + raise StopIteration + elif self.i >= len(self.params): + self.param_iter = None + self.p = None + self.i = None + block = to_bytes("--%s--\r\n" % self.boundary) + self.current += len(block) + if self.cb: + self.cb(self.p, self.current, self.total) + return block + + self.p = self.params[self.i] + self.param_iter = self.p.iter_encode(self.boundary) + self.i += 1 + return advance_iterator(self) + + def reset(self): + self.i = 0 + self.current = 0 + for param in self.params: + param.reset() + +def multipart_encode(params, boundary=None, cb=None): + """Encode ``params`` as multipart/form-data. + + ``params`` should be a sequence of (name, value) pairs or MultipartParam + objects, or a mapping of names to values. + Values are either strings parameter values, or file-like objects to use as + the parameter value. The file-like objects must support .read() and either + .fileno() or both .seek() and .tell(). + + If ``boundary`` is set, then it as used as the MIME boundary. Otherwise + a randomly generated boundary will be used. In either case, if the + boundary string appears in the parameter values a ValueError will be + raised. + + If ``cb`` is set, it should be a callback which will get called as blocks + of data are encoded. It will be called with (param, current, total), + indicating the current parameter being encoded, the current amount encoded, + and the total amount to encode. + + Returns a tuple of `datagen`, `headers`, where `datagen` is a + generator that will yield blocks of data that make up the encoded + parameters, and `headers` is a dictionary with the assoicated + Content-Type and Content-Length headers. + + Examples: + + >>> datagen, headers = multipart_encode( [("key", "value1"), ("key", "value2")] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> p = MultipartParam("key", "value2") + >>> datagen, headers = multipart_encode( [("key", "value1"), p] ) + >>> s = "".join(datagen) + >>> assert "value2" in s and "value1" in s + + >>> datagen, headers = multipart_encode( {"key": "value1"} ) + >>> s = "".join(datagen) + >>> assert "value2" not in s and "value1" in s + + """ + if boundary is None: + boundary = gen_boundary() + else: + boundary = quote_plus(boundary) + + headers = get_headers(params, boundary) + params = MultipartParam.from_params(params) + + return multipart_yielder(params, boundary, cb), headers diff --git a/scripts/cloudinary/poster/encode.pyc b/scripts/cloudinary/poster/encode.pyc new file mode 100644 index 0000000..4639555 Binary files /dev/null and b/scripts/cloudinary/poster/encode.pyc differ diff --git a/scripts/cloudinary/poster/streaminghttp.py b/scripts/cloudinary/poster/streaminghttp.py new file mode 100644 index 0000000..d8af521 --- /dev/null +++ b/scripts/cloudinary/poster/streaminghttp.py @@ -0,0 +1,201 @@ +# MIT licensed code copied from https://bitbucket.org/chrisatlee/poster +"""Streaming HTTP uploads module. + +This module extends the standard httplib and urllib2 objects so that +iterable objects can be used in the body of HTTP requests. + +In most cases all one should have to do is call :func:`register_openers()` +to register the new streaming http handlers which will take priority over +the default handlers, and then you can use iterable objects in the body +of HTTP requests. + +**N.B.** You must specify a Content-Length header if using an iterable object +since there is no way to determine in advance the total size that will be +yielded, and there is no way to reset an interator. + +Example usage: + +>>> from StringIO import StringIO +>>> import urllib2, poster.streaminghttp + +>>> opener = poster.streaminghttp.register_openers() + +>>> s = "Test file data" +>>> f = StringIO(s) + +>>> req = urllib2.Request("http://localhost:5000", f, +... {'Content-Length': str(len(s))}) +""" + +import sys, socket +from cloudinary.compat import httplib, urllib2, NotConnected + +__all__ = ['StreamingHTTPConnection', 'StreamingHTTPRedirectHandler', + 'StreamingHTTPHandler', 'register_openers'] + +if hasattr(httplib, 'HTTPS'): + __all__.extend(['StreamingHTTPSHandler', 'StreamingHTTPSConnection']) + +class _StreamingHTTPMixin: + """Mixin class for HTTP and HTTPS connections that implements a streaming + send method.""" + def send(self, value): + """Send ``value`` to the server. + + ``value`` can be a string object, a file-like object that supports + a .read() method, or an iterable object that supports a .next() + method. + """ + # Based on python 2.6's httplib.HTTPConnection.send() + if self.sock is None: + if self.auto_open: + self.connect() + else: + raise NotConnected() + + # send the data to the server. if we get a broken pipe, then close + # the socket. we want to reconnect when somebody tries to send again. + # + # NOTE: we DO propagate the error, though, because we cannot simply + # ignore the error... the caller will know if they can retry. + if self.debuglevel > 0: + print("send:", repr(value)) + try: + blocksize = 8192 + if hasattr(value, 'read') : + if hasattr(value, 'seek'): + value.seek(0) + if self.debuglevel > 0: + print("sendIng a read()able") + data = value.read(blocksize) + while data: + self.sock.sendall(data) + data = value.read(blocksize) + elif hasattr(value, 'next'): + if hasattr(value, 'reset'): + value.reset() + if self.debuglevel > 0: + print("sendIng an iterable") + for data in value: + self.sock.sendall(data) + else: + self.sock.sendall(value) + except socket.error: + e = sys.exc_info()[1] + if e[0] == 32: # Broken pipe + self.close() + raise + +class StreamingHTTPConnection(_StreamingHTTPMixin, httplib.HTTPConnection): + """Subclass of `httplib.HTTPConnection` that overrides the `send()` method + to support iterable body objects""" + +class StreamingHTTPRedirectHandler(urllib2.HTTPRedirectHandler): + """Subclass of `urllib2.HTTPRedirectHandler` that overrides the + `redirect_request` method to properly handle redirected POST requests + + This class is required because python 2.5's HTTPRedirectHandler does + not remove the Content-Type or Content-Length headers when requesting + the new resource, but the body of the original request is not preserved. + """ + + handler_order = urllib2.HTTPRedirectHandler.handler_order - 1 + + # From python2.6 urllib2's HTTPRedirectHandler + def redirect_request(self, req, fp, code, msg, headers, newurl): + """Return a Request or None in response to a redirect. + + This is called by the http_error_30x methods when a + redirection response is received. If a redirection should + take place, return a new Request to allow http_error_30x to + perform the redirect. Otherwise, raise HTTPError if no-one + else should try to handle this url. Return None if you can't + but another Handler might. + """ + m = req.get_method() + if (code in (301, 302, 303, 307) and m in ("GET", "HEAD") + or code in (301, 302, 303) and m == "POST"): + # Strictly (according to RFC 2616), 301 or 302 in response + # to a POST MUST NOT cause a redirection without confirmation + # from the user (of urllib2, in this case). In practice, + # essentially all clients do redirect in this case, so we + # do the same. + # be conciliant with URIs containing a space + newurl = newurl.replace(' ', '%20') + newheaders = dict((k, v) for k, v in req.headers.items() + if k.lower() not in ( + "content-length", "content-type") + ) + return urllib2.Request(newurl, + headers=newheaders, + origin_req_host=req.get_origin_req_host(), + unverifiable=True) + else: + raise urllib2.HTTPError(req.get_full_url(), code, msg, headers, fp) + +class StreamingHTTPHandler(urllib2.HTTPHandler): + """Subclass of `urllib2.HTTPHandler` that uses + StreamingHTTPConnection as its http connection class.""" + + handler_order = urllib2.HTTPHandler.handler_order - 1 + + def http_open(self, req): + """Open a StreamingHTTPConnection for the given request""" + return self.do_open(StreamingHTTPConnection, req) + + def http_request(self, req): + """Handle a HTTP request. Make sure that Content-Length is specified + if we're using an interable value""" + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPHandler.do_request_(self, req) + +if hasattr(httplib, 'HTTPS'): + class StreamingHTTPSConnection(_StreamingHTTPMixin, + httplib.HTTPSConnection): + """Subclass of `httplib.HTTSConnection` that overrides the `send()` + method to support iterable body objects""" + + class StreamingHTTPSHandler(urllib2.HTTPSHandler): + """Subclass of `urllib2.HTTPSHandler` that uses + StreamingHTTPSConnection as its http connection class.""" + + handler_order = urllib2.HTTPSHandler.handler_order - 1 + + def https_open(self, req): + return self.do_open(StreamingHTTPSConnection, req) + + def https_request(self, req): + # Make sure that if we're using an iterable object as the request + # body, that we've also specified Content-Length + if req.has_data(): + data = req.get_data() + if hasattr(data, 'read') or hasattr(data, 'next'): + if not req.has_header('Content-length'): + raise ValueError( + "No Content-Length specified for iterable body") + return urllib2.HTTPSHandler.do_request_(self, req) + + +def get_handlers(): + handlers = [StreamingHTTPHandler, StreamingHTTPRedirectHandler] + if hasattr(httplib, "HTTPS"): + handlers.append(StreamingHTTPSHandler) + return handlers + +def register_openers(): + """Register the streaming http handlers in the global urllib2 default + opener object. + + Returns the created OpenerDirector object.""" + opener = urllib2.build_opener(*get_handlers()) + + urllib2.install_opener(opener) + + return opener diff --git a/scripts/cloudinary/poster/streaminghttp.pyc b/scripts/cloudinary/poster/streaminghttp.pyc new file mode 100644 index 0000000..7441770 Binary files /dev/null and b/scripts/cloudinary/poster/streaminghttp.pyc differ diff --git a/scripts/cloudinary/templates/cloudinary_direct_upload.html b/scripts/cloudinary/templates/cloudinary_direct_upload.html new file mode 100644 index 0000000..2bf710c --- /dev/null +++ b/scripts/cloudinary/templates/cloudinary_direct_upload.html @@ -0,0 +1,12 @@ +
+ {% for name, value in params.items %} + + {% endfor %} + {% block extra %} {% endblock %} + {% block file %} + + {% endblock %} + {% block submit %} + + {% endblock %} +
diff --git a/scripts/cloudinary/templates/cloudinary_includes.html b/scripts/cloudinary/templates/cloudinary_includes.html new file mode 100644 index 0000000..be1fd15 --- /dev/null +++ b/scripts/cloudinary/templates/cloudinary_includes.html @@ -0,0 +1,14 @@ +{% load staticfiles %} + + + + + + +{% if processing %} + + + + + +{% endif %} diff --git a/scripts/cloudinary/templates/cloudinary_js_config.html b/scripts/cloudinary/templates/cloudinary_js_config.html new file mode 100644 index 0000000..dc7f489 --- /dev/null +++ b/scripts/cloudinary/templates/cloudinary_js_config.html @@ -0,0 +1,3 @@ + diff --git a/scripts/cloudinary/templatetags/__init__.py b/scripts/cloudinary/templatetags/__init__.py new file mode 100644 index 0000000..792d600 --- /dev/null +++ b/scripts/cloudinary/templatetags/__init__.py @@ -0,0 +1 @@ +# diff --git a/scripts/cloudinary/templatetags/cloudinary.py b/scripts/cloudinary/templatetags/cloudinary.py new file mode 100644 index 0000000..b7afd0f --- /dev/null +++ b/scripts/cloudinary/templatetags/cloudinary.py @@ -0,0 +1,67 @@ +from __future__ import absolute_import + +import json + +from django import template +from django.forms import Form + +import cloudinary +from cloudinary import CloudinaryImage, utils, uploader +from cloudinary.forms import CloudinaryJsFileField, cl_init_js_callbacks + +register = template.Library() + +@register.simple_tag(takes_context=True) +def cloudinary_url(context, source, options_dict={}, **options): + options = dict(options_dict, **options) + try: + if context['request'].is_secure() and 'secure' not in options: + options['secure'] = True + except KeyError: + pass + if not isinstance(source, CloudinaryImage): + source = CloudinaryImage(source) + return source.build_url(**options) + +@register.simple_tag(name='cloudinary', takes_context=True) +def cloudinary_tag(context, image, options_dict={}, **options): + options = dict(options_dict, **options) + try: + if context['request'].is_secure() and 'secure' not in options: + options['secure'] = True + except KeyError: + pass + if not isinstance(image, CloudinaryImage): + image = CloudinaryImage(image) + return image.image(**options) + +@register.simple_tag +def cloudinary_direct_upload_field(field_name="image", request=None): + form = type("OnTheFlyForm", (Form,), {field_name : CloudinaryJsFileField() })() + if request: + cl_init_js_callbacks(form, request) + return unicode(form[field_name]) + +"""Deprecated - please use cloudinary_direct_upload_field, or a proper form""" +@register.inclusion_tag('cloudinary_direct_upload.html') +def cloudinary_direct_upload(callback_url, **options): + params = utils.build_upload_params(callback=callback_url, **options) + params = utils.sign_request(params, options) + + api_url = utils.cloudinary_api_url("upload", resource_type=options.get("resource_type", "image"), upload_prefix=options.get("upload_prefix")) + + return {"params": params, "url": api_url} + +@register.inclusion_tag('cloudinary_includes.html') +def cloudinary_includes(processing=False): + return {"processing": processing} + +CLOUDINARY_JS_CONFIG_PARAMS = ("api_key", "cloud_name", "private_cdn", "secure_distribution", "cdn_subdomain") +@register.inclusion_tag('cloudinary_js_config.html') +def cloudinary_js_config(): + config = cloudinary.config() + return dict( + params = json.dumps(dict( + (param, getattr(config, param)) for param in CLOUDINARY_JS_CONFIG_PARAMS if getattr(config, param, None) + )) + ) diff --git a/scripts/cloudinary/uploader.py b/scripts/cloudinary/uploader.py new file mode 100644 index 0000000..aa1d002 --- /dev/null +++ b/scripts/cloudinary/uploader.py @@ -0,0 +1,234 @@ +# Copyright Cloudinary +import json, re, sys +from os.path import basename +import urllib +import cloudinary +import socket +from cloudinary import utils +from cloudinary.api import Error +from cloudinary.poster.encode import multipart_encode +from cloudinary.poster.streaminghttp import register_openers +from cloudinary.compat import urllib2, BytesIO, string_types, urlencode, to_bytes, to_string, PY3, HTTPError +_initialized = False + +def upload(file, **options): + params = utils.build_upload_params(**options) + return call_api("upload", params, file = file, **options) + +def unsigned_upload(file, upload_preset, **options): + return upload(file, upload_preset=upload_preset, unsigned=True, **options) + +def upload_image(file, **options): + result = upload(file, **options) + return cloudinary.CloudinaryImage(result["public_id"], version=str(result["version"]), + format=result.get("format"), metadata=result) + +def upload_large(file, **options): + """ Upload large raw files. Note that public_id should include an extension for best results. """ + with open(file, 'rb') as file_io: + upload = upload_id = None + index = 1 + public_id = options.get("public_id") + chunk = file_io.read(20000000) + while (chunk): + chunk_io = BytesIO(chunk) + chunk_io.name = basename(file) + chunk = file_io.read(20000000) + upload = upload_large_part(chunk_io, public_id=public_id, + upload_id=upload_id, part_number=index, final=chunk == "", **options) + upload_id = upload.get("upload_id") + public_id = upload.get("public_id") + index += 1 + return upload + +def upload_large_part(file, **options): + """ Upload large raw files. Note that public_id should include an extension for best results. """ + params = { + "timestamp": utils.now(), + "type": options.get("type"), + "backup": options.get("backup"), + "final": options.get("final"), + "part_number": options.get("part_number"), + "upload_id": options.get("upload_id"), + "tags": options.get("tags") and ",".join(utils.build_array(options["tags"])), + "public_id": options.get("public_id") + } + return call_api("upload_large", params, resource_type="raw", file=file, **options) + + +def destroy(public_id, **options): + params = { + "timestamp": utils.now(), + "type": options.get("type"), + "invalidate": options.get("invalidate"), + "public_id": public_id + } + return call_api("destroy", params, **options) + +def rename(from_public_id, to_public_id, **options): + params = { + "timestamp": utils.now(), + "type": options.get("type"), + "overwrite": options.get("overwrite"), + "from_public_id": from_public_id, + "to_public_id": to_public_id + } + return call_api("rename", params, **options) + +def explicit(public_id, **options): + params = { + "timestamp": utils.now(), + "type": options.get("type"), + "public_id": public_id, + "callback": options.get("callback"), + "headers": utils.build_custom_headers(options.get("headers")), + "eager": utils.build_eager(options.get("eager")), + "tags": options.get("tags") and ",".join(utils.build_array(options["tags"])), + "face_coordinates": utils.encode_double_array(options.get("face_coordinates")), + "custom_coordinates": utils.encode_double_array(options.get("custom_coordinates"))} + return call_api("explicit", params, **options) + +def generate_sprite(tag, **options): + params = { + "timestamp": utils.now(), + "tag": tag, + "async": options.get("async"), + "notification_url": options.get("notification_url"), + "transformation": utils.generate_transformation_string(fetch_format=options.get("format"), **options)[0] + } + return call_api("sprite", params, **options) + +def multi(tag, **options): + params = { + "timestamp": utils.now(), + "tag": tag, + "format": options.get("format"), + "async": options.get("async"), + "notification_url": options.get("notification_url"), + "transformation": utils.generate_transformation_string(**options)[0] + } + return call_api("multi", params, **options) + +def explode(public_id, **options): + params = { + "timestamp": utils.now(), + "public_id": public_id, + "format": options.get("format"), + "notification_url": options.get("notification_url"), + "transformation": utils.generate_transformation_string(**options)[0] + } + return call_api("explode", params, **options) + +# options may include 'exclusive' (boolean) which causes clearing this tag from all other resources +def add_tag(tag, public_ids = [], **options): + exclusive = options.pop("exclusive", None) + command = "set_exclusive" if exclusive else "add" + return call_tags_api(tag, command, public_ids, **options) + +def remove_tag(tag, public_ids = [], **options): + return call_tags_api(tag, "remove", public_ids, **options) + +def replace_tag(tag, public_ids = [], **options): + return call_tags_api(tag, "replace", public_ids, **options) + +def call_tags_api(tag, command, public_ids = [], **options): + params = { + "timestamp": utils.now(), + "tag": tag, + "public_ids": utils.build_array(public_ids), + "command": command, + "type": options.get("type") + } + return call_api("tags", params, **options) + +TEXT_PARAMS = ["public_id", "font_family", "font_size", "font_color", "text_align", "font_weight", "font_style", "background", "opacity", "text_decoration"] +def text(text, **options): + params = {"timestamp": utils.now(), "text": text} + for key in TEXT_PARAMS: + params[key] = options.get(key) + return call_api("text", params, **options) + +def call_api(action, params, **options): + try: + file_io = None + return_error = options.get("return_error") + if options.get("unsigned"): + params = utils.cleanup_params(params) + else: + params = utils.sign_request(params, options) + + param_list = [] + for k, v in params.items(): + if isinstance(v, list): + for vv in v: + param_list.append((k+"[]", vv)) + elif v: + param_list.append((k, v)) + + api_url = utils.cloudinary_api_url(action, **options) + + global _initialized + if not _initialized: + _initialized = True + # Register the streaming http handlers with urllib2 + register_openers() + + datagen = [] + headers = {} + if "file" in options: + file = options["file"] + if not isinstance(file, string_types): + datagen, headers = multipart_encode({'file': file}) + elif not re.match(r'^https?:|^s3:|^data:[^;]*;base64,([a-zA-Z0-9\/+\n=]+)$', file): + file_io = open(file, "rb") + datagen, headers = multipart_encode({'file': file_io}) + else: + param_list.append(("file", file)) + + if _is_gae(): + # Might not be needed in the future but for now this is needed in GAE + datagen = "".join(datagen) + + request = urllib2.Request(api_url + "?" + urlencode(param_list), datagen, headers) + request.add_header("User-Agent", cloudinary.USER_AGENT) + + kw = {} + if 'timeout' in options: + kw['timeout'] = options['timeout'] + + code = 200 + try: + response = urllib2.urlopen(request, **kw).read() + except HTTPError: + e = sys.exc_info()[1] + if not e.code in [200, 400, 500]: + raise Error("Server returned unexpected status code - %d - %s" % (e.code, e.read())) + code = e.code + response = e.read() + except socket.error: + e = sys.exc_info()[1] + raise Error("Socket error: %s" % str(e)) + + try: + result = json.loads(to_string(response)) + except Exception: + e = sys.exc_info()[1] + # Error is parsing json + raise Error("Error parsing server response (%d) - %s. Got - %s", code, response, e) + + if "error" in result: + if return_error: + result["error"]["http_code"] = code + else: + raise Error(result["error"]["message"]) + + return result + finally: + if file_io: file_io.close() + +def _is_gae(): + if PY3: + return False + else: + import httplib + return 'appengine' in str(httplib.HTTP) diff --git a/scripts/cloudinary/uploader.pyc b/scripts/cloudinary/uploader.pyc new file mode 100644 index 0000000..d992bca Binary files /dev/null and b/scripts/cloudinary/uploader.pyc differ diff --git a/scripts/cloudinary/utils.py b/scripts/cloudinary/utils.py new file mode 100644 index 0000000..8374abe --- /dev/null +++ b/scripts/cloudinary/utils.py @@ -0,0 +1,373 @@ +# Copyright Cloudinary +import zlib, hashlib, re, struct, uuid, base64, time +import cloudinary +from cloudinary.compat import (PY3, to_bytes, to_bytearray, to_string, unquote, urlencode) + +""" @deprecated: use cloudinary.SHARED_CDN """ +SHARED_CDN = cloudinary.SHARED_CDN + +DEFAULT_RESPONSIVE_WIDTH_TRANSFORMATION = {"width": "auto", "crop": "limit"} + +def build_array(arg): + if isinstance(arg, list): + return arg + elif arg == None: + return [] + else: + return [arg] + +def encode_double_array(array): + array = build_array(array) + if len(array) > 0 and isinstance(array[0], list): + return "|".join([",".join([str(i) for i in build_array(inner)]) for inner in array]) + else: + return ",".join([str(i) for i in array]) + +def encode_dict(arg): + if isinstance(arg, dict): + if PY3: + items = arg.items() + else: + items = arg.iteritems() + return "|".join((k + "=" + v) for k, v in items) + else: + return arg + +def generate_transformation_string(**options): + responsive_width = options.pop("responsive_width", cloudinary.config().responsive_width) + size = options.pop("size", None) + if size: + options["width"], options["height"] = size.split("x") + width = options.get("width") + height = options.get("height") + has_layer = ("underlay" in options) or ("overlay" in options) + + crop = options.pop("crop", None) + angle = ".".join([str(value) for value in build_array(options.pop("angle", None))]) + no_html_sizes = has_layer or angle or crop == "fit" or crop == "limit" or responsive_width + + if width and (width == "auto" or float(width) < 1 or no_html_sizes): + del options["width"] + if height and (float(height) < 1 or no_html_sizes): + del options["height"] + + background = options.pop("background", None) + if background: + background = background.replace("#", "rgb:") + color = options.pop("color", None) + if color: + color = color.replace("#", "rgb:") + + base_transformations = build_array(options.pop("transformation", None)) + if any(isinstance(bs, dict) for bs in base_transformations): + recurse = lambda bs: generate_transformation_string(**bs)[0] if isinstance(bs, dict) else generate_transformation_string(transformation=bs)[0] + base_transformations = list(map(recurse, base_transformations)) + named_transformation = None + else: + named_transformation = ".".join(base_transformations) + base_transformations = [] + + effect = options.pop("effect", None) + if isinstance(effect, list): + effect = ":".join([str(x) for x in effect]) + elif isinstance(effect, dict): + effect = ":".join([str(x) for x in list(effect.items())[0]]) + + border = options.pop("border", None) + if isinstance(border, dict): + border = "%(width)spx_solid_%(color)s" % {"color": border.get("color", "black").replace("#", "rgb:"), "width": str(border.get("width", 2))} + + flags = ".".join(build_array(options.pop("flags", None))) + dpr = options.pop("dpr", cloudinary.config().dpr) + + params = {"w": width, "h": height, "t": named_transformation, "b": background, "co": color, "e": effect, "c": crop, "a": angle, "bo": border, "fl": flags, "dpr": dpr} + for param, option in {"q": "quality", "g": "gravity", "p": "prefix", "x": "x", + "y": "y", "r": "radius", "d": "default_image", "l": "overlay", "u": "underlay", "o": "opacity", + "f": "fetch_format", "pg": "page", "dn": "density", "dl": "delay", "cs": "color_space"}.items(): + params[param] = options.pop(option, None) + + transformation = ",".join(sorted([param + "_" + str(value) for param, value in params.items() if (value or value == 0)])) + if "raw_transformation" in options: + transformation = transformation + "," + options.pop("raw_transformation") + transformations = base_transformations + [transformation] + if responsive_width: + responsive_width_transformation = cloudinary.config().responsive_width_transformation or DEFAULT_RESPONSIVE_WIDTH_TRANSFORMATION + transformations += [generate_transformation_string(**responsive_width_transformation)[0]] + url = "/".join([trans for trans in transformations if trans]) + + if width == "auto" or responsive_width: + options["responsive"] = True + if dpr == "auto": + options["hidpi"] = True + return (url, options) + +def cleanup_params(params): + return dict( [ (k, __safe_value(v)) for (k,v) in params.items() if not v is None and not v == ""] ) + +def sign_request(params, options): + api_key = options.get("api_key", cloudinary.config().api_key) + if not api_key: raise ValueError("Must supply api_key") + api_secret = options.get("api_secret", cloudinary.config().api_secret) + if not api_secret: raise ValueError("Must supply api_secret") + + params = cleanup_params(params) + params["signature"] = api_sign_request(params, api_secret) + params["api_key"] = api_key + + return params + +def api_sign_request(params_to_sign, api_secret): + to_sign = "&".join(sorted([(k+"="+(",".join(v) if isinstance(v, list) else str(v))) for k, v in params_to_sign.items() if v])) + return hashlib.sha1(to_bytes(to_sign + api_secret)).hexdigest() + +def finalize_source(source, format, url_suffix): + source = re.sub(r'([^:])/+', r'\1/', source) + if re.match(r'^https?:/', source): + source = smart_escape(source) + source_to_sign = source + else: + source = unquote(source) + if not PY3: source = source.decode('utf8') + source = smart_escape(source) + source_to_sign = source + if url_suffix != None: + if re.search(r'[\./]', url_suffix): raise ValueError("url_suffix should not include . or /") + source = source + "/" + url_suffix + if format != None: + source = source + "." + format + source_to_sign = source_to_sign + "." + format + + return (source, source_to_sign) + +def finalize_resource_type(resource_type, type, url_suffix, use_root_path, shorten): + type = type or "upload" + if url_suffix != None: + if resource_type == "image" and type == "upload": + resource_type = "images" + type = None + elif resource_type == "raw" and type == "upload": + resource_type = "files" + type = None + else: + raise ValueError("URL Suffix only supported for image/upload and raw/upload") + + if use_root_path: + if (resource_type == "image" and type == "upload") or (resource_type == "images" and type == None): + resource_type = None + type = None + else: + raise ValueError("Root path only supported for image/upload") + + if shorten and resource_type == "image" and type == "upload": + resource_type = "iu" + type = None + + return (resource_type, type) + +def unsigned_download_url_prefix(source, cloud_name, private_cdn, cdn_subdomain, secure_cdn_subdomain, cname, secure, secure_distribution): + """cdn_subdomain and secure_cdn_subdomain + 1) Customers in shared distribution (e.g. res.cloudinary.com) + if cdn_domain is true uses res-[1-5].cloudinary.com for both http and https. Setting secure_cdn_subdomain to false disables this for https. + 2) Customers with private cdn + if cdn_domain is true uses cloudname-res-[1-5].cloudinary.com for http + if secure_cdn_domain is true uses cloudname-res-[1-5].cloudinary.com for https (please contact support if you require this) + 3) Customers with cname + if cdn_domain is true uses a[1-5].cname for http. For https, uses the same naming scheme as 1 for shared distribution and as 2 for private distribution.""" + shared_domain = not private_cdn + shard = __crc(source) + if secure: + if secure_distribution == None or secure_distribution == cloudinary.OLD_AKAMAI_SHARED_CDN: + secure_distribution = cloud_name + "-res.cloudinary.com" if private_cdn else cloudinary.SHARED_CDN + + shared_domain = shared_domain or secure_distribution == cloudinary.SHARED_CDN + if secure_cdn_subdomain == None and shared_domain: + secure_cdn_subdomain = cdn_subdomain + + if secure_cdn_subdomain: + secure_distribution = re.sub('res.cloudinary.com', "res-" + shard + ".cloudinary.com", secure_distribution) + + prefix = "https://" + secure_distribution + elif cname: + subdomain = "a" + shard + "." if cdn_subdomain else "" + prefix = "http://" + subdomain + cname + else: + subdomain = cloud_name + "-res" if private_cdn else "res" + if cdn_subdomain: subdomain = subdomain + "-" + shard + prefix = "http://" + subdomain + ".cloudinary.com" + + if shared_domain: prefix += "/" + cloud_name + + return prefix + +def cloudinary_url(source, **options): + original_source = source + + type = options.pop("type", "upload") + if type == 'fetch': + options["fetch_format"] = options.get("fetch_format", options.pop("format", None)) + transformation, options = generate_transformation_string(**options) + + resource_type = options.pop("resource_type", "image") + version = options.pop("version", None) + format = options.pop("format", None) + cdn_subdomain = options.pop("cdn_subdomain", cloudinary.config().cdn_subdomain) + secure_cdn_subdomain = options.pop("secure_cdn_subdomain", cloudinary.config().secure_cdn_subdomain) + cname = options.pop("cname", cloudinary.config().cname) + shorten = options.pop("shorten", cloudinary.config().shorten) + + cloud_name = options.pop("cloud_name", cloudinary.config().cloud_name or None) + if cloud_name is None: + raise ValueError("Must supply cloud_name in tag or in configuration") + secure = options.pop("secure", cloudinary.config().secure) + private_cdn = options.pop("private_cdn", cloudinary.config().private_cdn) + secure_distribution = options.pop("secure_distribution", cloudinary.config().secure_distribution) + sign_url = options.pop("sign_url", cloudinary.config().sign_url) + api_secret = options.pop("api_secret", cloudinary.config().api_secret) + url_suffix = options.pop("url_suffix", None) + use_root_path = options.pop("use_root_path", cloudinary.config().use_root_path) + + if url_suffix and not private_cdn: + raise ValueError("URL Suffix only supported in private CDN") + + + if (not source) or type == "upload" and re.match(r'^https?:', source): + return (original_source, options) + + resource_type, type = finalize_resource_type(resource_type, type, url_suffix, use_root_path, shorten) + source, source_to_sign = finalize_source(source, format, url_suffix) + + + if source_to_sign.find("/") >= 0 and not re.match(r'^https?:/', source_to_sign) and not re.match(r'^v[0-9]+', source_to_sign) and not version: + version = "1" + if version: version = "v" + str(version) + + transformation = re.sub(r'([^:])/+', r'\1/', transformation) + + signature = None + if sign_url: + to_sign = "/".join(__compact([transformation, source_to_sign])) + signature = "s--" + to_string(base64.urlsafe_b64encode( hashlib.sha1(to_bytes(to_sign + api_secret)).digest() )[0:8]) + "--" + + prefix = unsigned_download_url_prefix(source, cloud_name, private_cdn, cdn_subdomain, secure_cdn_subdomain, cname, secure, secure_distribution) + source = "/".join(__compact([prefix, resource_type, type, signature, transformation, version, source])) + return (source, options) + +def cloudinary_api_url(action = 'upload', **options): + cloudinary_prefix = options.get("upload_prefix", cloudinary.config().upload_prefix) or "https://api.cloudinary.com" + cloud_name = options.get("cloud_name", cloudinary.config().cloud_name) + if not cloud_name: raise ValueError("Must supply cloud_name") + resource_type = options.get("resource_type", "image") + return "/".join([cloudinary_prefix, "v1_1", cloud_name, resource_type, action]) + +# Based on ruby's CGI::unescape. In addition does not escape / : +def smart_escape(string): + pack = lambda m: to_bytes('%' + "%".join(["%02X" % x for x in struct.unpack('B'*len(m.group(1)), m.group(1))]).upper()) + return to_string(re.sub(to_bytes(r"([^a-zA-Z0-9_.\-\/:]+)"), pack, to_bytes(string))) + +def random_public_id(): + return base64.urlsafe_b64encode(hashlib.sha1(uuid.uuid4()).digest())[0:16] + +def signed_preloaded_image(result): + filename = ".".join([x for x in [result["public_id"], result["format"]] if x]) + path = "/".join([result["resource_type"], "upload", "v" + result["version"], filename]) + return path + "#" + result["signature"] + +def now(): + return str(int(time.time())) + +def private_download_url(public_id, format, **options): + cloudinary_params = sign_request({ + "timestamp": now(), + "public_id": public_id, + "format": format, + "type": options.get("type"), + "attachment": options.get("attachment"), + "expires_at": options.get("expires_at") + }, options) + + return cloudinary_api_url("download", **options) + "?" + urlencode(cloudinary_params) + +def zip_download_url(tag, **options): + cloudinary_params = sign_request({ + "timestamp": now(), + "tag": tag, + "transformation": generate_transformation_string(**options)[0] + }, options) + + return cloudinary_api_url("download_tag.zip", **options) + "?" + urlencode(cloudinary_params) + +def build_eager(transformations): + eager = [] + for tr in build_array(transformations): + format = tr.get("format") + single_eager = "/".join([x for x in [generate_transformation_string(**tr)[0], format] if x]) + eager.append(single_eager) + return "|".join(eager) + +def build_custom_headers(headers): + if headers == None: + return None + elif isinstance(headers, list): + pass + elif isinstance(headers, dict): + headers = [k + ": " + v for k, v in headers.items()] + else: + return headers + return "\n".join(headers) + +def build_upload_params(**options): + params = {"timestamp": now(), + "transformation": generate_transformation_string(**options)[0], + "public_id": options.get("public_id"), + "callback": options.get("callback"), + "format": options.get("format"), + "type": options.get("type"), + "backup": options.get("backup"), + "faces": options.get("faces"), + "image_metadata": options.get("image_metadata"), + "exif": options.get("exif"), + "colors": options.get("colors"), + "headers": build_custom_headers(options.get("headers")), + "eager": build_eager(options.get("eager")), + "use_filename": options.get("use_filename"), + "unique_filename": options.get("unique_filename"), + "discard_original_filename": options.get("discard_original_filename"), + "invalidate": options.get("invalidate"), + "notification_url": options.get("notification_url"), + "eager_notification_url": options.get("eager_notification_url"), + "eager_async": options.get("eager_async"), + "proxy": options.get("proxy"), + "folder": options.get("folder"), + "overwrite": options.get("overwrite"), + "tags": options.get("tags") and ",".join(build_array(options["tags"])), + "allowed_formats": options.get("allowed_formats") and ",".join(build_array(options["allowed_formats"])), + "face_coordinates": encode_double_array(options.get("face_coordinates")), + "custom_coordinates": encode_double_array(options.get("custom_coordinates")), + "context": encode_dict(options.get("context")), + "moderation": options.get("moderation"), + "raw_convert": options.get("raw_convert"), + "ocr": options.get("ocr"), + "categorization": options.get("categorization"), + "detection": options.get("detection"), + "similarity_search": options.get("similarity_search"), + "background_removal": options.get("background_removal"), + "upload_preset": options.get("upload_preset"), + "phash": options.get("phash"), + "return_delete_token": options.get("return_delete_token"), + "auto_tagging": options.get("auto_tagging") and float(options.get("auto_tagging"))} + return params + +def __safe_value(v): + if isinstance(v, (bool)): + if v: + return "1" + else: + return "0" + else: + return v +def __crc(source): + return str((zlib.crc32(to_bytearray(source)) & 0xffffffff)%5 + 1) + +def __compact(array): + return filter(lambda x: x, array) + diff --git a/scripts/cloudinary/utils.pyc b/scripts/cloudinary/utils.pyc new file mode 100644 index 0000000..d658a75 Binary files /dev/null and b/scripts/cloudinary/utils.pyc differ