From c32e6d36fdd3a6bb53b6b45fedb089e96a4249f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Maro=C5=A1i?= Date: Wed, 10 Nov 2021 11:26:01 +0100 Subject: [PATCH 1/2] Use akamai cli fast purge to refresh cache (#58) --- src/Jenkinsfile | 2 +- src/akamai_cache_buster/bustCache.py | 141 +++++------------------ src/akamai_cache_buster/requirements.txt | 17 --- 3 files changed, 33 insertions(+), 127 deletions(-) diff --git a/src/Jenkinsfile b/src/Jenkinsfile index 224c350..409049c 100644 --- a/src/Jenkinsfile +++ b/src/Jenkinsfile @@ -65,7 +65,7 @@ node { withCredentials([file(credentialsId: "jenkins-eccu-cache-purge", variable: 'EDGERC')]) { //path to .edgerc file is now set to $EDGERC" //Bust the current cache - sh "python3 bustCache.py $EDGERC ${APP_NAME}" + sh "python3 bustCache.py $EDGERC ${APP_NAME} ${BRANCH}" } } } diff --git a/src/akamai_cache_buster/bustCache.py b/src/akamai_cache_buster/bustCache.py index 5224551..45ebca3 100644 --- a/src/akamai_cache_buster/bustCache.py +++ b/src/akamai_cache_buster/bustCache.py @@ -1,11 +1,8 @@ import yaml -import json import sys -import os -import configparser +import subprocess import requests from urllib.parse import urljoin -from akamai.edgegrid import EdgeGridAuth, EdgeRc # Set up connectivity. Global var because it's a session that's used in multiple functions. s = requests.Session() @@ -14,112 +11,16 @@ def getYMLFromUrl(url): return yaml.safe_load(s.get(url).content.decode('utf-8')) -# Initializes the EdgeGrid auth using the .edgerc file (or some passed-in config). -def initEdgeGridAuth(path="~/.edgerc"): - # If the config file was passed in, use that. - if len(sys.argv) > 1: - path = sys.argv[1] - config = configparser.RawConfigParser() - config.read(os.path.expanduser(path)) - - # TODO: We might actually be able to authenticate without EdgeGridAuth, - # which would reduce the number of dependencies. - s.auth = EdgeGridAuth( - client_token=config.get("default", "client_token"), - client_secret=config.get("default", "client_secret"), - access_token=config.get("default", "access_token") -) - -def akamaiPost(url, body): - return s.post(urljoin(base_url, url), json=body).content - -# Gets the hostname from the .edgerc file (or some passed-in config). -def getHostFromConfig(path="~/.edgerc"): - # If the config file was passed in, use that. - if len(sys.argv) > 1: - path = sys.argv[1] - config = configparser.RawConfigParser() - config.read(os.path.expanduser(path)) - return config.get("default", "host") - -# Get the base url using the provided config -base_url = "https://" + getHostFromConfig() - -#uses the paths for the app and the environments it's released on to generate -#the XML used in the API request -def createMetadata(paths, releases, appName): - #Add the begining XML - metadata = '\n\n\n' - - #generate the paths XML - for key in releases: - # generate JS/CSS assets paths - prefix = releases[key].get("content_path_prefix") - if (prefix == None): - prefix = '/' - splitPrefix = f"apps{prefix}/{appName}".split('/') - splitPrefix = list(filter(len, splitPrefix)) - splitPrefixLength = len(splitPrefix) - closingTag = '' - for i in range(0, splitPrefixLength): - metadata += ' ' * i + f'\n' - closingTag += ' ' * (splitPrefixLength - i - 1) +'\n' - metadata += ' ' * (i + 1) + 'now\n' - metadata += closingTag - # generate HTML paths - for path in paths: - path = prefix + path - splitPath = path.split('/') - splitPath = list(filter(len, splitPath)) - metadataClosingTags = '' - pathLength = len(splitPath) - #create opening and closing tags - - for i in range(0, pathLength): - metadata += ' ' * i + f'\n' - metadataClosingTags += ' ' * (pathLength - i - 1) + '\n' - metadata += ' ' * pathLength + 'now\n' - metadata += metadataClosingTags - # generate chrome JSON config paths - prefix = releases[key].get("content_path_prefix") - if (prefix == None): - prefix = '' - chromeConfigPath = f'{prefix}/config/chrome'.split('/') - chromeSplitPath = list(filter(len, chromeConfigPath)) - chromeSplitPathLen = len(chromeSplitPath) - metadataClosingTags = '' - for i in range(0, chromeSplitPathLen): - metadata += ' ' * i + f'\n' - metadataClosingTags += ' ' * (chromeSplitPathLen - i - 1) +'\n' - metadata += ' ' * (i + 1) + 'now\n' - metadata += metadataClosingTags - metadata += '' - - return metadata - -def createRequest(paths, releases, appName): - body = { - "propertyName": "cloud.redhat.com", - "propertyNameExactMatch": 'true', - "propertyType": "HOST_HEADER", - "metadata": createMetadata(paths, releases, appName), - "notes": "purging cache for new deployment", - "requestName": f"Invalidate cache for {appName}", - "statusUpdateEmails": [ - "rfelton@redhat.com", - "fms-alerts@redhat.com" - ] - } - - return body - #main def main(): + edgeRcPath = sys.argv[1] appName = sys.argv[2] - - #connect to akamai and validate - initEdgeGridAuth() - + branch = sys.argv[3] + domain = 'https://console.stage.redhat.com' + if 'prod' in branch: + domain = 'https://console.redhat.com' + entryBase = f'/apps/{appName}' + fedModsBase = f'{entryBase}/fed-mods.json' #get the data to use for cache busting paths = [] try: @@ -127,10 +28,32 @@ def main(): except: print("WARNING: this app has no path, if that's okay ignore this :)") paths = [] - + releases = getYMLFromUrl("https://console.redhat.com/config/releases.yml") - akamaiPost("/eccu-api/v1/requests", createRequest(paths, releases, appName)) + print(paths) + purgeSuffixes = [] + purgeUrls = [] + for key in releases: + prefix = releases[key].get("content_path_prefix") + if (prefix == None): + prefix = '' + purgeSuffixes.append(f'{prefix}{fedModsBase}') + for path in paths: + purgeSuffixes.append(f'{prefix}{path}') + + for suffix in purgeSuffixes: + purgeUrls.append(f'{domain}{suffix}') + + for endpoint in purgeUrls: + print(f'Purging endpoint cache: {endpoint}') + try: + subprocess.check_output(['akamai', 'purge', '--edgerc', edgeRcPath , 'invalidate', endpoint]) + except subprocess.CalledProcessError as e: + print(e.output) + sys.exit(1) + + if __name__ == "__main__": main() \ No newline at end of file diff --git a/src/akamai_cache_buster/requirements.txt b/src/akamai_cache_buster/requirements.txt index f2a4c47..fa294fd 100644 --- a/src/akamai_cache_buster/requirements.txt +++ b/src/akamai_cache_buster/requirements.txt @@ -1,20 +1,3 @@ -asn1crypto==0.24.0 -certifi==2019.3.9 -cffi==1.12.3 -chardet==3.0.4 -cryptography==3.2 -edgegrid-python==1.1.1 -enum34==1.1.6 -httpie==1.0.3 -httpie-edgegrid==1.0.2 -idna==2.8 -ipaddress==1.0.22 -ndg-httpsclient==0.5.1 -pyasn1==0.4.5 -pycparser==2.19 -Pygments==2.4.0 -pyOpenSSL==19.0.0 PyYAML==5.1 requests==2.21.0 -six==1.12.0 urllib3==1.24.2 From 068d536b544a871767f83e457f7678b81f70b1f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 10 Nov 2021 10:27:09 +0000 Subject: [PATCH 2/2] Bump urllib3 from 1.24.2 to 1.26.5 in /src/akamai_cache_buster Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.24.2 to 1.26.5. - [Release notes](https://github.com/urllib3/urllib3/releases) - [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst) - [Commits](https://github.com/urllib3/urllib3/compare/1.24.2...1.26.5) --- updated-dependencies: - dependency-name: urllib3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- src/akamai_cache_buster/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/akamai_cache_buster/requirements.txt b/src/akamai_cache_buster/requirements.txt index fa294fd..1d768d1 100644 --- a/src/akamai_cache_buster/requirements.txt +++ b/src/akamai_cache_buster/requirements.txt @@ -1,3 +1,3 @@ PyYAML==5.1 requests==2.21.0 -urllib3==1.24.2 +urllib3==1.26.5