diff --git a/.pre-commit-config.py27.yaml b/.pre-commit-config.py27.yaml index 292566d..0149127 100644 --- a/.pre-commit-config.py27.yaml +++ b/.pre-commit-config.py27.yaml @@ -32,22 +32,19 @@ repos: args: - --max-line-length=79 - --ignore-imports=yes + - -d bad-continuation - -d fixme - -d import-error - -d invalid-name - -d locally-disabled - -d missing-docstring - -d too-few-public-methods + - -d too-many-arguments - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 hooks: - id: remove-crlf - id: remove-tabs -- repo: https://github.com/asottile/reorder_python_imports - rev: v1.5.0 - hooks: - - id: reorder-python-imports - args: [--py26-plus] - repo: https://github.com/asottile/yesqa rev: v0.0.11 hooks: @@ -60,5 +57,4 @@ repos: rev: v1.4.0 hooks: - id: python-no-eval - - id: python-no-log-warn - - id: python-use-type-annotations + # - id: python-no-log-warn TODO: fix the hook to not catch 'warnings.warn' and uncomment once done diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 64d8be5..75486f7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,12 +32,14 @@ repos: args: - --max-line-length=79 - --ignore-imports=yes + - -d duplicate-code - -d fixme - -d import-error - -d invalid-name - -d locally-disabled - -d missing-docstring - -d too-few-public-methods + - -d too-many-arguments - -d useless-object-inheritance # necessary for Python 2 compatibility - repo: https://github.com/Lucas-C/pre-commit-hooks rev: v1.1.6 @@ -73,8 +75,7 @@ repos: rev: v1.4.0 hooks: - id: python-no-eval - - id: python-no-log-warn - - id: python-use-type-annotations + # - id: python-no-log-warn TODO: fix the hook to not catch 'warnings.warn' and uncomment once done - id: rst-backticks - repo: https://github.com/Lucas-C/pre-commit-hooks-markup rev: v1.0.0 diff --git a/README.rst b/README.rst index 689719a..2e337df 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ RESTful Google Cloud Client Library for Python ============================================== This project is a collection of Google Cloud client libraries for the REST-only -APIs; its *raison d'être* is to implement a simple `CloudTasks API`_ as well as +APIs; its *raison d'etre* is to implement a simple `CloudTasks API`_ as well as a more abstract TaskManager. If you don't need to support Python 2, you probably want to use `gcloud-aio`_, diff --git a/gcloud/rest/auth/__init__.py b/gcloud/rest/auth/__init__.py index 743ca65..3591306 100644 --- a/gcloud/rest/auth/__init__.py +++ b/gcloud/rest/auth/__init__.py @@ -1,7 +1,10 @@ from pkg_resources import get_distribution __version__ = get_distribution('gcloud-rest').version +from gcloud.rest.auth.iam import IamClient from gcloud.rest.auth.token import Token +from gcloud.rest.auth.utils import decode +from gcloud.rest.auth.utils import encode -__all__ = ['__version__', 'Token'] +__all__ = ['__version__', 'IamClient', 'Token', 'decode', 'encode'] diff --git a/gcloud/rest/auth/iam.py b/gcloud/rest/auth/iam.py new file mode 100644 index 0000000..4392636 --- /dev/null +++ b/gcloud/rest/auth/iam.py @@ -0,0 +1,147 @@ +import json +import threading +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import +from typing import Union # pylint: disable=unused-import + +import requests + +from .token import Token +from .token import Type +from .utils import encode + + +API_ROOT_IAM = 'https://iam.googleapis.com/v1' +API_ROOT_IAM_CREDENTIALS = 'https://iamcredentials.googleapis.com/v1' +SCOPES = ['https://www.googleapis.com/auth/iam'] + + +class IamClient(object): + def __init__(self, + service_file=None, # type: Optional[str] + session=None, # type: Optional[requests.Session] + google_api_lock=None, # type: Optional[threading.RLock] + token=None # type: Optional[Token] + ): + # type: (...) -> None + self.session = session + self.google_api_lock = google_api_lock or threading.RLock() + self.token = token or Token(service_file=service_file, + session=session, scopes=SCOPES) + + if self.token.token_type != Type.SERVICE_ACCOUNT: + raise TypeError('IAM Credentials Client is only valid for use' + ' with Service Accounts') + + def headers(self): + # type: () -> Dict[str, str] + token = self.token.get() + return { + 'Authorization': 'Bearer {}'.format(token), + } + + @property + def service_account_email(self): + # type: () -> Optional[str] + return self.token.service_data.get('client_email') + + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts.keys/get + def get_public_key(self, + key_id=None, # type: Optional[str] + key=None, # type: Optional[str] + service_account_email=None, # type: Optional[str] + project=None, # type: Optional[str] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> Dict[str, str] + service_account_email = (service_account_email + or self.service_account_email) + project = project or self.token.get_project() + + if not key_id and not key: + raise ValueError('get_public_key must have either key_id or key') + + if not key: + key = 'projects/{}/serviceAccounts/{}/keys/{}' \ + .format(project, service_account_email, key_id) + + url = '{}/{}?publicKeyType=TYPE_X509_PEM_FILE'.format( + API_ROOT_IAM, key) + headers = self.headers() + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.get(url, headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json() + + # https://cloud.google.com/iam/reference/rest/v1/projects.serviceAccounts.keys/list + def list_public_keys(self, + service_account_email=None, # type: Optional[str] + project=None, # type: Optional[str] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> List[Dict[str, str]] + service_account_email = (service_account_email + or self.service_account_email) + project = project or self.token.get_project() + + url = ('{}/projects/{}/serviceAccounts/' + '{}/keys').format(API_ROOT_IAM, project, service_account_email) + + headers = self.headers() + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.get(url, headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json().get('keys', []) + + # https://cloud.google.com/iam/credentials/reference/rest/v1/projects.serviceAccounts/signBlob + def sign_blob(self, + payload, # type: Optional[Union[str, bytes]] + service_account_email=None, # type: Optional[str] + delegates=None, # type: Optional[list] + session=None, # type: requests.Session + timeout=10 # type: int + ): + # type: (...) -> Dict[str, str] + service_account_email = ( + service_account_email or self.service_account_email) + if not service_account_email: + raise TypeError('sign_blob must have a valid ' + 'service_account_email') + + resource_name = 'projects/-/serviceAccounts/{}'.format( + service_account_email) + url = '{}/{}:signBlob'.format(API_ROOT_IAM_CREDENTIALS, resource_name) + + json_str = json.dumps({ + 'delegates': delegates or [resource_name], + 'payload': encode(payload).decode('utf-8'), + }) + + headers = self.headers() + headers.update({ + 'Content-Length': str(len(json_str)), + 'Content-Type': 'application/json', + }) + + if not self.session: + self.session = requests.Session() + + session = session or self.session + with self.google_api_lock: + resp = session.post(url, data=json_str, + headers=headers, timeout=timeout) + resp.raise_for_status() + return resp.json() diff --git a/gcloud/rest/auth/token.py b/gcloud/rest/auth/token.py index cbe8514..8991861 100644 --- a/gcloud/rest/auth/token.py +++ b/gcloud/rest/auth/token.py @@ -1,14 +1,22 @@ +"""Google Cloud auth via service account file""" import datetime +import enum import json -import logging import os import threading import time +import warnings +from typing import Any # pylint: disable=unused-import +from typing import Dict # pylint: disable=unused-import +from typing import List # pylint: disable=unused-import +from typing import Optional # pylint: disable=unused-import + try: from urllib.parse import urlencode except ImportError: from urllib import urlencode +import backoff # N.B. the cryptography library is required when calling jwt.encrypt() with # algorithm='RS256'. It does not need to be imported here, but this allows us # to throw this error at load time rather than lazily during normal operations, @@ -19,77 +27,208 @@ import requests -TOKEN_URI = 'https://accounts.google.com/o/oauth2/token' -TIMEOUT = 60 - -log = logging.getLogger(__name__) +GCE_METADATA_BASE = 'http://metadata.google.internal/computeMetadata/v1' +GCE_METADATA_HEADERS = {'metadata-flavor': 'Google'} +GCE_ENDPOINT_PROJECT = '{}/project/project-id'.format(GCE_METADATA_BASE) +GCE_ENDPOINT_TOKEN = \ + '{}/instance/service-accounts/default/token?recursive=true'\ + .format(GCE_METADATA_BASE) +GCLOUD_TOKEN_DURATION = 3600 +REFRESH_HEADERS = {'Content-Type': 'application/x-www-form-urlencoded'} + + +class Type(enum.Enum): + AUTHORIZED_USER = 'authorized_user' + GCE_METADATA = 'gce_metadata' + SERVICE_ACCOUNT = 'service_account' + + +def get_service_data(service): + # type: (Optional[str]) -> Dict[str, Any] + service = service or os.environ.get('GOOGLE_APPLICATION_CREDENTIALS') + if not service: + cloudsdk_config = os.environ.get('CLOUDSDK_CONFIG') + sdkpath = (cloudsdk_config + or os.path.join(os.path.expanduser('~'), '.config', + 'gcloud')) + service = os.path.join(sdkpath, 'application_default_credentials.json') + set_explicitly = bool(cloudsdk_config) + else: + set_explicitly = True + + try: + with open(service, 'r') as f: + data = json.loads(f.read()) + return data + except (IOError, OSError): + # only warn users if they have explicitly set the service_file path + if set_explicitly: + raise + return {} + except Exception: # pylint: disable=broad-except + return {} class Token(object): - def __init__(self, creds=None, google_api_lock=None, scopes=None, - timeout=TIMEOUT): - self.creds = creds or os.getenv('GOOGLE_APPLICATION_CREDENTIALS') - if not self.creds: - raise Exception('could not load service credentials') + # pylint: disable=too-many-instance-attributes + def __init__(self, + creds=None, # type: Optional[str] + google_api_lock=None, # type: Optional[threading.RLock] + scopes=None, # type: Optional[List[str]] + timeout=None, # type: Optional[int] + service_file=None, # type: Optional[str] + session=None, # type: Optional[requests.Session] + ): + # type: (...) -> None + if creds: + warnings.warn('creds is now deprecated for Token(),' + 'please use service_file instead', + DeprecationWarning) + service_file = creds + if timeout: + warnings.warn( + 'timeout arg is now deprecated for Token()', + DeprecationWarning) + + self.service_data = get_service_data(service_file) + if self.service_data: + self.token_type = Type(self.service_data['type']) + self.token_uri = self.service_data.get( + 'token_uri', 'https://oauth2.googleapis.com/token') + else: + # At this point, all we can do is assume we're running somewhere + # with default credentials, eg. GCE. + self.token_type = Type.GCE_METADATA + self.token_uri = GCE_ENDPOINT_TOKEN self.google_api_lock = google_api_lock or threading.RLock() - self.scopes = scopes or [] - self.timeout = timeout - - self.age = datetime.datetime.now() - self.expiry = 60 - self.value = None + self.session = session + self.scopes = ' '.join(scopes or []) + if self.token_type == Type.SERVICE_ACCOUNT and not self.scopes: + raise Exception('scopes must be provided when token type is ' + 'service account') + + self.access_token = None + self.access_token_duration = 0 + self.access_token_acquired_at = datetime.datetime(1970, 1, 1) + + self.acquiring = None + + def get_project(self): + # type: () -> Optional[str] + project = (os.environ.get('GOOGLE_CLOUD_PROJECT') + or os.environ.get('GCLOUD_PROJECT') + or os.environ.get('APPLICATION_ID')) + + if self.token_type == Type.GCE_METADATA: + self.ensure_token() + with self.google_api_lock: + resp = self.session.get(GCE_ENDPOINT_PROJECT, timeout=10, + headers=GCE_METADATA_HEADERS) + resp.raise_for_status() + project = project or resp.text + elif self.token_type == Type.SERVICE_ACCOUNT: + project = project or self.service_data.get('project_id') + + return project + + def get(self): + # type: () -> str + self.ensure_token() + return self.access_token def __str__(self): - self.ensure() - return str(self.value) - - def assertion(self): - with open(self.creds, 'r') as f: - credentials = json.loads(f.read()) - - # N.B. the below exists to avoid using this private method: - # return ServiceAccountCredentials._generate_assertion() - now = int(time.time()) - payload = { - 'aud': TOKEN_URI, - 'exp': now + 3600, - 'iat': now, - 'iss': credentials['client_email'], - 'scope': ' '.join(self.scopes), - } - - return jwt.encode(payload, credentials['private_key'], - algorithm='RS256') + # type: () -> str + return str(self.get()) def acquire(self): - headers = {'Content-Type': 'application/x-www-form-urlencoded'} - body = urlencode(( - ('grant_type', 'urn:ietf:params:oauth:grant-type:jwt-bearer'), - ('assertion', self.assertion()), - )) - - with self.google_api_lock: - response = requests.post(TOKEN_URI, data=body, headers=headers, - timeout=self.timeout) + # type: () -> str + warnings.warn('Token.acquire() is deprecated', + 'please use Token.acquire_access_token()', + DeprecationWarning) + return self.acquire_access_token() + + def ensure_token(self): + # type: () -> None + if not self.access_token: + self.acquire_access_token() + return - content = response.json() - if 'error' in content: - raise Exception('{}'.format(content)) + now = datetime.datetime.utcnow() + delta = (now - self.access_token_acquired_at).total_seconds() + if delta <= self.access_token_duration / 2: + return - self.age = datetime.datetime.now() - self.expiry = int(content['expires_in']) - self.value = content['access_token'] + self.acquire_access_token() def ensure(self): - if not self.value: - log.debug('acquiring initial token') - self.acquire() - return + # type: () -> None + warnings.warn('Token.ensure() is deprecated', + 'please use Token.ensure_token()', + DeprecationWarning) + self.ensure_token() + + def _refresh_authorized_user(self, timeout): + # type: (int) -> requests.Response + payload = urlencode({ + 'grant_type': 'refresh_token', + 'client_id': self.service_data['client_id'], + 'client_secret': self.service_data['client_secret'], + 'refresh_token': self.service_data['refresh_token'], + }) + with self.google_api_lock: + return self.session.post(self.token_uri, data=payload, + headers=REFRESH_HEADERS, timeout=timeout) + + def _refresh_gce_metadata(self, timeout): + # type: (int) -> requests.Response + with self.google_api_lock: + return self.session.get(self.token_uri, + headers=GCE_METADATA_HEADERS, + timeout=timeout) - now = datetime.datetime.now() - delta = (now - self.age).total_seconds() + def _refresh_service_account(self, timeout): + # type: (int) -> requests.Response + now = int(time.time()) + assertion_payload = { + 'aud': self.token_uri, + 'exp': now + GCLOUD_TOKEN_DURATION, + 'iat': now, + 'iss': self.service_data['client_email'], + 'scope': self.scopes, + } - if delta > self.expiry / 2: - log.debug('requiring token with expiry %d of %d / 2', delta, - self.expiry) - self.acquire() + # N.B. algorithm='RS256' requires an extra 240MB in dependencies... + assertion = jwt.encode(assertion_payload, + self.service_data['private_key'], + algorithm='RS256') + payload = urlencode({ + 'assertion': assertion, + 'grant_type': 'urn:ietf:params:oauth:grant-type:jwt-bearer', + }) + with self.google_api_lock: + return self.session.post(self.token_uri, data=payload, + headers=REFRESH_HEADERS, timeout=timeout) + + @backoff.on_exception(backoff.expo, Exception, max_tries=5) # type: ignore + def acquire_access_token(self, timeout=10): + # type: (int) -> None + if not self.session: + self.session = requests.Session() + + if self.token_type == Type.AUTHORIZED_USER: + resp = self._refresh_authorized_user(timeout=timeout) + elif self.token_type == Type.GCE_METADATA: + resp = self._refresh_gce_metadata(timeout=timeout) + elif self.token_type == Type.SERVICE_ACCOUNT: + resp = self._refresh_service_account(timeout=timeout) + else: + raise Exception( + 'unsupported token type {}'.format(self.token_type)) + + resp.raise_for_status() + content = resp.json() + + self.access_token = str(content['access_token']) + self.access_token_duration = int(content['expires_in']) + self.access_token_acquired_at = datetime.datetime.utcnow() + self.acquiring = None diff --git a/gcloud/rest/auth/utils.py b/gcloud/rest/auth/utils.py new file mode 100644 index 0000000..a34a655 --- /dev/null +++ b/gcloud/rest/auth/utils.py @@ -0,0 +1,25 @@ +import base64 +from typing import Union # pylint: disable=unused-import + + +def decode(payload): + # type: (str) -> bytes + """ + Modified Base64 for URL variants exist, where the + and / characters of + standard Base64 are respectively replaced by - and _. + See https://en.wikipedia.org/wiki/Base64#URL_applications + """ + return base64.b64decode(payload, altchars=b'-_') + + +def encode(payload): + # type: (Union[bytes, str]) -> bytes + """ + Modified Base64 for URL variants exist, where the + and / characters of + standard Base64 are respectively replaced by - and _. + See https://en.wikipedia.org/wiki/Base64#URL_applications + """ + if isinstance(payload, str): + payload = payload.encode('utf-8') + + return base64.b64encode(payload, altchars=b'-_') diff --git a/gcloud/rest/kms/client.py b/gcloud/rest/kms/client.py index 0bdc1ed..30b0d6b 100644 --- a/gcloud/rest/kms/client.py +++ b/gcloud/rest/kms/client.py @@ -22,7 +22,7 @@ def __init__(self, project, keyring, keyname, creds=None, self.google_api_lock = google_api_lock or threading.RLock() - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/gcloud/rest/storage/bucket.py b/gcloud/rest/storage/bucket.py index a857645..28e2dc0 100644 --- a/gcloud/rest/storage/bucket.py +++ b/gcloud/rest/storage/bucket.py @@ -22,8 +22,7 @@ def __init__(self, project, bucket, creds=None, google_api_lock=None): self.bucket = bucket self.google_api_lock = google_api_lock or threading.RLock() - - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/gcloud/rest/taskqueue/queue.py b/gcloud/rest/taskqueue/queue.py index e5f5893..e4721d4 100644 --- a/gcloud/rest/taskqueue/queue.py +++ b/gcloud/rest/taskqueue/queue.py @@ -21,7 +21,7 @@ def __init__(self, project, taskqueue, creds=None, google_api_lock=None, self.google_api_lock = google_api_lock or threading.RLock() - self.access_token = Token(creds=creds, + self.access_token = Token(service_file=creds, google_api_lock=self.google_api_lock, scopes=SCOPES) diff --git a/requirements.txt b/requirements.txt index 0cc917b..620363a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +backoff >= 1.0.0, < 2.0.0 cryptography >= 2.0.0, < 3.0.0 pyjwt >= 1.5.3, < 2.0.0 requests[security] >= 2.0.0, < 3.0.0 +typing >= 3.0.0, < 4.0.0 diff --git a/tests/unit/auth/iam_test.py b/tests/unit/auth/iam_test.py new file mode 100644 index 0000000..f5aef16 --- /dev/null +++ b/tests/unit/auth/iam_test.py @@ -0,0 +1,5 @@ +import gcloud.rest.auth.iam as iam # pylint: disable=unused-import + + +def test_importable(): + assert True diff --git a/tests/unit/auth/utils_test.py b/tests/unit/auth/utils_test.py new file mode 100644 index 0000000..1515520 --- /dev/null +++ b/tests/unit/auth/utils_test.py @@ -0,0 +1,19 @@ +import pickle + +import pytest + +import gcloud.rest.auth.utils as utils + + +@pytest.mark.parametrize('str_or_bytes', ['Hello Test', + 'UTF-8 Bytes'.encode('utf-8'), + pickle.dumps([])]) +def test_encode_decode(str_or_bytes): + encoded = utils.encode(str_or_bytes) + expected = str_or_bytes + if isinstance(expected, str): + try: + expected = str_or_bytes.encode('utf-8') + except UnicodeDecodeError: + pass + assert expected == utils.decode(encoded)