diff --git a/.github/scripts/determine-hub-deployments.py b/.github/scripts/determine-hub-deployments.py index 3172858b5..43d876589 100755 --- a/.github/scripts/determine-hub-deployments.py +++ b/.github/scripts/determine-hub-deployments.py @@ -11,15 +11,19 @@ If no hubs need deploying, nothing will be emitted. """ + import argparse import os + def main(args): hubs = [] # Deploy all hubs by getting deployment names from the dirs in deployments/ - if "GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT" in os.environ.keys() or \ - "GITHUB_PR_LABEL_HUB_IMAGES" in os.environ.keys(): + if ( + "GITHUB_PR_LABEL_JUPYTERHUB_DEPLOYMENT" in os.environ.keys() + or "GITHUB_PR_LABEL_HUB_IMAGES" in os.environ.keys() + ): for deployment in next(os.walk(args.deployments))[1]: if deployment not in args.ignore: hubs.append(deployment) @@ -27,8 +31,7 @@ def main(args): # Deploy only the modified/flagged hubs by PR labels else: hub_labels = [ - k.lower() for k in os.environ.keys() - if k.startswith("GITHUB_PR_LABEL_HUB_") + k.lower() for k in os.environ.keys() if k.startswith("GITHUB_PR_LABEL_HUB_") ] hubs = [x.split("_")[-1] for x in hub_labels] hubs = [x for x in hubs if x not in args.ignore] @@ -39,6 +42,7 @@ def main(args): continue print(h) + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Get hubs that need to be deployed from environment variables." @@ -47,7 +51,7 @@ def main(args): "--deployments", "-d", default="deployments", - help="The directory to search for deployments." + help="The directory to search for deployments.", ) parser.add_argument( "--ignore", @@ -55,13 +59,10 @@ def main(args): nargs="*", action="extend", default=["template"], - help="Ignore one or more deployment targets." + help="Ignore one or more deployment targets.", ) parser.add_argument( - "--only-deploy", - "-o", - nargs="*", - help="Only deploy the specified hubs." + "--only-deploy", "-o", nargs="*", help="Only deploy the specified hubs." ) args = parser.parse_args() diff --git a/hub/Chart.yaml b/hub/Chart.yaml index 20df2ce2c..5559675a6 100644 --- a/hub/Chart.yaml +++ b/hub/Chart.yaml @@ -2,4 +2,4 @@ apiVersion: v1 appVersion: '1.0' description: Deployment Chart for JupyterHub name: hub -version: 20240731-224556.git.8059.hf384f3d1 +version: 20240731-224556.git.8607.hf7abb041 diff --git a/images/node-placeholder-scaler/scaler/__main__.py b/images/node-placeholder-scaler/scaler/__main__.py index b5169df4f..bfb5c7513 100644 --- a/images/node-placeholder-scaler/scaler/__main__.py +++ b/images/node-placeholder-scaler/scaler/__main__.py @@ -2,4 +2,4 @@ if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/images/node-placeholder-scaler/scaler/calendar.py b/images/node-placeholder-scaler/scaler/calendar.py index 95ba45efa..d494620f4 100755 --- a/images/node-placeholder-scaler/scaler/calendar.py +++ b/images/node-placeholder-scaler/scaler/calendar.py @@ -9,6 +9,7 @@ log = logging.getLogger(__name__) + def _event_repr(event): """ Simple repr of a calenar event @@ -24,6 +25,7 @@ def _event_repr(event): else: return f"{event.summary} {event.start.strftime('%Y-%m-%d %H:%M %Z')} to {event.end.strftime('%Y-%m-%d %H:%M %Z')}" + def _get_cal_tz(calendar): """ Get the calendar timezone. @@ -38,6 +40,7 @@ def _get_cal_tz(calendar): else: return zoneinfo.ZoneInfo("UTC") + def get_calendar(url: str): """ Get a calendar from local file or URL. @@ -62,6 +65,7 @@ def get_calendar(url: str): else: logging.error(f"Unable to get calendar from resource: {url}") + def get_events(calendar, time=None): """ Get events from a calendar. If no time is passed, assume now. @@ -80,6 +84,6 @@ def get_events(calendar, time=None): # https://stackoverflow.com/questions/753052/strip-html-from-strings-in-python events = [x for x in events_iter] for ev in events: - ev.description = re.sub('<[^<]+?>', '', ev.description) + ev.description = re.sub("<[^<]+?>", "", ev.description) return events diff --git a/images/node-placeholder-scaler/scaler/scaler.py b/images/node-placeholder-scaler/scaler/scaler.py index ea2e79af9..8265618bf 100755 --- a/images/node-placeholder-scaler/scaler/scaler.py +++ b/images/node-placeholder-scaler/scaler/scaler.py @@ -11,6 +11,7 @@ from .calendar import get_calendar, get_events, _event_repr from ruamel.yaml import YAML + yaml = YAML(typ="safe") @@ -54,7 +55,9 @@ def get_replica_counts(events): try: pools_replica_config = yaml.load(ev.description) except Exception as e: - logging.error(f"Caught unhandled exception parsing event description:\n{e}") + logging.error( + f"Caught unhandled exception parsing event description:\n{e}" + ) logging.error(f"Error in parsing description of {_event_repr(ev)}") logging.error(f"{ev.description=}") pass diff --git a/node-placeholder/Chart.yaml b/node-placeholder/Chart.yaml index 51222e273..912579940 100644 --- a/node-placeholder/Chart.yaml +++ b/node-placeholder/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 20240731-224556.git.8187.he9c6025c +version: 20240731-224556.git.8610.hedc17750 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/node-placeholder/values.yaml b/node-placeholder/values.yaml index 0416f33d8..6dada1924 100644 --- a/node-placeholder/values.yaml +++ b/node-placeholder/values.yaml @@ -4,7 +4,7 @@ image: repository: us-central1-docker.pkg.dev/ucb-datahub-2018/core/node-placeholder-scaler pullPolicy: IfNotPresent # Overrides the image tag whose default is the chart appVersion. - tag: "20240731-224556.git.8187.he9c6025c" + tag: "20240731-224556.git.8610.hedc17750" imagePullSecrets: [] nameOverride: "" diff --git a/requirements.txt b/requirements.txt index cf410a6e9..88193e5d1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,3 +10,4 @@ chardet niquests==3.7.2 urllib3<2.0.0 yamllint==1.35.1 +pre-commit==4.0.1 diff --git a/scripts/delete-unused-users.py b/scripts/delete-unused-users.py index 6922a7185..f13687cc4 100755 --- a/scripts/delete-unused-users.py +++ b/scripts/delete-unused-users.py @@ -14,6 +14,7 @@ Core functionality from @minrk: https://discourse.jupyter.org/t/is-there-a-way-to-bulk-delete-old-users/20866/3 """ + import argparse from datetime import timedelta, datetime import json @@ -36,8 +37,8 @@ def parse_timedelta(args): https://docs.python.org/3/library/datetime.html#datetime.timedelta """ result = {} - for arg in args.split(','): - key, value = arg.split('=') + for arg in args.split(","): + key, value = arg.split("=") try: value = int(value) except ValueError: @@ -48,6 +49,7 @@ def parse_timedelta(args): result[key] = value return timedelta(**result) + def retrieve_users(hub_url, headers, inactive_since): """Returns generator of user models that should be deleted""" url = hub_url.rstrip("/") + "/hub/api/users" @@ -72,36 +74,44 @@ def retrieve_users(hub_url, headers, inactive_since): "limit": next_page["limit"], } + def should_delete(user, inactive_since): """ Returns a boolean if user is to be deleted. The critera are: - was the user active in the past inactive_since period? - is there a current user server running? """ - last_activity_str = user.get('last_activity', False) + last_activity_str = user.get("last_activity", False) if last_activity_str: try: - last_activity = parse(user['last_activity']) + last_activity = parse(user["last_activity"]) except: - logger.error(f"Unexpected value for user['last_activity']: {user['last_activity']}") + logger.error( + f"Unexpected value for user['last_activity']: {user['last_activity']}" + ) raise if isinstance(last_activity, datetime): - was_active_recently = datetime.now().astimezone() - last_activity < inactive_since + was_active_recently = ( + datetime.now().astimezone() - last_activity < inactive_since + ) else: - logger.error(f"For user {user['name']}, expected datetime.datetime class for last_activity but got {type(last_activity)} instead.") + logger.error( + f"For user {user['name']}, expected datetime.datetime class for last_activity but got {type(last_activity)} instead." + ) raise logger.debug(f"User: {user['name']}") logger.debug(f"Last login: {last_activity}") logger.debug(f"Recent activity: {was_active_recently}") logger.debug(f"Running server: {user['server']}") - if was_active_recently or user['server'] is not None: + if was_active_recently or user["server"] is not None: logger.info(f"Not flagging {user['name']} for deletion.") return False else: logger.info(f"Flagging {user['name']} for deletion.") return True + def delete_user(hub_url, headers, name): """Delete a given user by name via JupyterHub API""" r = niquests.delete( @@ -110,6 +120,7 @@ def delete_user(hub_url, headers, name): ) r.raise_for_status() + def delete_users_from_hub(hub_url, token, inactive_since, dry_run=False): """Delete users from a provided hub url""" headers = { @@ -124,7 +135,7 @@ def delete_users_from_hub(hub_url, token, inactive_since, dry_run=False): count = 1 for user in users: if not dry_run: - delete_user(hub_url, headers, user['name']) + delete_user(hub_url, headers, user["name"]) logger.info(f"{count}: deleting {user['name']}") else: logger.info(f"Skipped {user['name']} due to dry run.") @@ -134,7 +145,10 @@ def delete_users_from_hub(hub_url, token, inactive_since, dry_run=False): if not dry_run: print(f"Deleted {count} total users from the ORM for hub: {hub_url}") else: - print(f"Dry run: Did not delete {count} total users from the ORM for hub: {hub_url}") + print( + f"Dry run: Did not delete {count} total users from the ORM for hub: {hub_url}" + ) + def main(args): """ @@ -142,11 +156,15 @@ def main(args): and if so, delete them! """ if args.credentials and args.hub_url: - logger.error("Please use only one of --hub_url or --credentials options when executing the script.") + logger.error( + "Please use only one of --hub_url or --credentials options when executing the script." + ) raise if args.hub_url: - logger.info(f"Checking for and deleting ORM users on a single hub: {args.hub_url}") + logger.info( + f"Checking for and deleting ORM users on a single hub: {args.hub_url}" + ) token = os.environ["JUPYTERHUB_API_TOKEN"] if not token: @@ -169,48 +187,44 @@ def main(args): print() else: - logger.error("You must specify a single hub with the --hub_url argument, or a json file containing hubs and api keys with the --credentials argument.") + logger.error( + "You must specify a single hub with the --hub_url argument, or a json file containing hubs and api keys with the --credentials argument." + ) raise if __name__ == "__main__": argparser = argparse.ArgumentParser() argparser.add_argument( - '-c', - '--credentials', - dest='credentials', - help='Path to a json file containing hub url and api keys. Format is: {"hub1_url": "hub1_key", "hub2_url":, "hub2_key"}' + "-c", + "--credentials", + dest="credentials", + help='Path to a json file containing hub url and api keys. Format is: {"hub1_url": "hub1_key", "hub2_url":, "hub2_key"}', ) argparser.add_argument( - '-H', - '--hub_url', - help='Fully qualified URL to the JupyterHub. You must also set the JUPYTERHUB_API_TOKEN environment variable with the API key.' + "-H", + "--hub_url", + help="Fully qualified URL to the JupyterHub. You must also set the JUPYTERHUB_API_TOKEN environment variable with the API key.", ) argparser.add_argument( - '--dry_run', - action='store_true', - help='Dry run without deleting users.' + "--dry_run", action="store_true", help="Dry run without deleting users." ) argparser.add_argument( - '--inactive_since', - default='hours=24', + "--inactive_since", + default="hours=24", type=parse_timedelta, - help='Period of inactivity after which users are considered for deletion (literal string constructor values for timedelta objects).' + help="Period of inactivity after which users are considered for deletion (literal string constructor values for timedelta objects).", # https://docs.python.org/3/library/datetime.html#timedelta-objects ) argparser.add_argument( - '-v', - '--verbose', - dest='verbose', - action='store_true', - help='Set info log level.' + "-v", + "--verbose", + dest="verbose", + action="store_true", + help="Set info log level.", ) argparser.add_argument( - '-d', - '--debug', - dest='debug', - action='store_true', - help='Set debug log level.' + "-d", "--debug", dest="debug", action="store_true", help="Set debug log level." ) args = argparser.parse_args() diff --git a/scripts/git-pre-cloner.py b/scripts/git-pre-cloner.py index 6c13466dd..667110958 100644 --- a/scripts/git-pre-cloner.py +++ b/scripts/git-pre-cloner.py @@ -8,26 +8,31 @@ import escapism safe_chars = set(string.ascii_lowercase + string.digits) -repo = 'https://github.com/data-8/materials-fa17.git' -local_repo = '/export/pool0/homes/_repo' -cwd_tmpl = '/export/pool0/homes/{}' +repo = "https://github.com/data-8/materials-fa17.git" +local_repo = "/export/pool0/homes/_repo" +cwd_tmpl = "/export/pool0/homes/{}" + def safe_username(username): - return escapism.escape(username, safe=safe_chars, escape_char='-').lower() + return escapism.escape(username, safe=safe_chars, escape_char="-").lower() + def home_directory(username): home_dir = cwd_tmpl.format(username) if not os.path.exists(home_dir): os.mkdir(home_dir) return home_dir - + + def git_clone(): if os.path.exists(os.path.join(local_repo, repo_dirname)): return - out = subprocess.check_output(['git', 'clone', args.repo], - cwd=local_repo).decode('utf-8') + out = subprocess.check_output(["git", "clone", args.repo], cwd=local_repo).decode( + "utf-8" + ) return out + def copy_repo(username): safe = safe_username(username) home_dir = home_directory(safe) @@ -35,25 +40,25 @@ def copy_repo(username): dest_dir = os.path.join(home_dir, repo_dirname) if os.path.exists(dest_dir): if args.verbose: - print('Skipping {}'.format(safe)) + print("Skipping {}".format(safe)) else: if args.verbose: print(safe) - out = subprocess.check_output(['cp', '-a', source_dir, dest_dir]) + out = subprocess.check_output(["cp", "-a", source_dir, dest_dir]) return out return + # main -parser = argparse.ArgumentParser(description='Pre-clone course assets.') -parser.add_argument('-f', dest='filename', required=True, - help='File containing user emails') -parser.add_argument('-r', dest='repo', default=repo, - help='Course asset repo') -parser.add_argument('-v', dest='verbose', action='store_true', - help='Be verbose.') +parser = argparse.ArgumentParser(description="Pre-clone course assets.") +parser.add_argument( + "-f", dest="filename", required=True, help="File containing user emails" +) +parser.add_argument("-r", dest="repo", default=repo, help="Course asset repo") +parser.add_argument("-v", dest="verbose", action="store_true", help="Be verbose.") args = parser.parse_args() -repo_dirname = os.path.basename(args.repo).split('.')[0] +repo_dirname = os.path.basename(args.repo).split(".")[0] if not os.path.exists(local_repo): os.mkdir(local_repo) @@ -64,11 +69,11 @@ def copy_repo(username): f = open(args.filename) line = f.readline() -while line != '': +while line != "": email = line.strip() - if '@berkeley.edu' not in email: - continue # just in case - username = email.split('@')[0] + if "@berkeley.edu" not in email: + continue # just in case + username = email.split("@")[0] copy_repo(username) line = f.readline() diff --git a/scripts/list-packages.py b/scripts/list-packages.py index f1f6a7da4..b825804ed 100755 --- a/scripts/list-packages.py +++ b/scripts/list-packages.py @@ -2,6 +2,7 @@ """ Lists R packages in one docker image but not the other """ + import docker import argparse import json @@ -10,11 +11,11 @@ argparser = argparse.ArgumentParser() argparser.add_argument( - 'src_image', + "src_image", ) argparser.add_argument( - 'dest_image', + "dest_image", ) args = argparser.parse_args() @@ -26,7 +27,7 @@ def get_package_info(package_name): """ Return package data for package_name in CRAN repo """ - url = f'https://packagemanager.rstudio.com/__api__/repos/1/packages/{package_name}' + url = f"https://packagemanager.rstudio.com/__api__/repos/1/packages/{package_name}" try: with urlopen(url) as resp: @@ -36,28 +37,34 @@ def get_package_info(package_name): if e.code == 404: # Package doesn't exist print(f'Package "{package_name}" not found in package manager') - return { "name": package_name, "version": None } + return {"name": package_name, "version": None} else: raise return data + def packages_list(image_name): - raw_packages = client.containers.run( - image_name, - 'R --quiet -e "installed.packages()[,c(1, 3)]"' - ).decode().split('\n')[2:] + raw_packages = ( + client.containers.run( + image_name, 'R --quiet -e "installed.packages()[,c(1, 3)]"' + ) + .decode() + .split("\n")[2:] + ) return set([rp.split()[0] for rp in raw_packages if len(rp.split()) == 3]) + def main(): src_packages = packages_list(args.src_image) dest_packages = packages_list(args.dest_image) - to_be_added = src_packages - dest_packages + to_be_added = src_packages - dest_packages for p in to_be_added: info = get_package_info(p) print(f'"{p}", "{info["version"]}",') -if __name__ == '__main__': + +if __name__ == "__main__": main() diff --git a/scripts/post-grafana-annotation.py b/scripts/post-grafana-annotation.py index bc161834c..888555b90 100644 --- a/scripts/post-grafana-annotation.py +++ b/scripts/post-grafana-annotation.py @@ -12,11 +12,13 @@ with at least Editor permissions - The requests library """ + import argparse import niquests as requests import os import time + def create_annotation(grafana_url, grafana_api_key, tags, text): """ Create annotation in a grafana instance. @@ -24,44 +26,36 @@ def create_annotation(grafana_url, grafana_api_key, tags, text): return requests.post( grafana_url + "/api/annotations", json={ - 'tags': tags, - 'text': text, - 'time': int(time.time() * 1000), - 'isRegion': False + "tags": tags, + "text": text, + "time": int(time.time() * 1000), + "isRegion": False, }, - headers={ - 'Authorization': f'Bearer {grafana_api_key}' - } + headers={"Authorization": f"Bearer {grafana_api_key}"}, ).text def main(): argparser = argparse.ArgumentParser() - argparser.add_argument( - '--grafana-url', - help='URL of the grafana instance to use' - ) + argparser.add_argument("--grafana-url", help="URL of the grafana instance to use") argparser.add_argument( - '--tag', - help='Tags to add to the annotation', + "--tag", + help="Tags to add to the annotation", default=[], - action='append', - dest='tags' + action="append", + dest="tags", ) - argparser.add_argument( - 'text', - help='Text to use for the annotation' - ) + argparser.add_argument("text", help="Text to use for the annotation") args = argparser.parse_args() - print(create_annotation( - args.grafana_url, - os.environ['GRAFANA_API_KEY'], - args.tags, - args.text - )) + print( + create_annotation( + args.grafana_url, os.environ["GRAFANA_API_KEY"], args.tags, args.text + ) + ) + -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/scripts/rsync-active-users.py b/scripts/rsync-active-users.py index db4afb2a1..797402a64 100755 --- a/scripts/rsync-active-users.py +++ b/scripts/rsync-active-users.py @@ -9,6 +9,7 @@ An environment variable 'JUPYTERHUB_ADMIN' must be set with an admin token, obtainable from {hub_url}/hub/token by an admin user. """ + import os import niquests as requests from dateutil.parser import parse @@ -25,14 +26,16 @@ import sys SAFE = set(string.ascii_letters + string.digits) -ESCAPE_CHAR = '_' +ESCAPE_CHAR = "_" + + def _escape_char(c, escape_char=ESCAPE_CHAR): """Escape a single character""" buf = [] - for byte in c.encode('utf8'): + for byte in c.encode("utf8"): buf.append(escape_char) - buf.append('%X' % byte) - return ''.join(buf) + buf.append("%X" % byte) + return "".join(buf) def escape(to_escape, safe=SAFE, escape_char=ESCAPE_CHAR, allow_collisions=False): @@ -58,7 +61,7 @@ def escape(to_escape, safe=SAFE, escape_char=ESCAPE_CHAR, allow_collisions=False """ if isinstance(to_escape, bytes): # always work on text - to_escape = to_escape.decode('utf8') + to_escape = to_escape.decode("utf8") if not isinstance(safe, set): safe = set(safe) @@ -81,81 +84,80 @@ def escape(to_escape, safe=SAFE, escape_char=ESCAPE_CHAR, allow_collisions=False chars.append(c) else: chars.append(_escape_char(c, escape_char)) - return u''.join(chars) - + return "".join(chars) def get_all_users(hub_url, token): - url = f'{hub_url}/hub/api/users' - resp = requests.get(url, headers={ - 'Authorization': f'token {token}' - }) + url = f"{hub_url}/hub/api/users" + resp = requests.get(url, headers={"Authorization": f"token {token}"}) users = resp.json() for user in users: - if user['last_activity']: - user['last_activity'] = parse(user.get('last_activity')) + if user["last_activity"]: + user["last_activity"] = parse(user.get("last_activity")) return users def rsync(user, src_basedir, dest_basedir, dry_run): start_time = time.perf_counter() safe_chars = set(string.ascii_lowercase + string.digits) - homedir = escape(user, safe_chars, '-').lower() + homedir = escape(user, safe_chars, "-").lower() src_homedir = os.path.join(src_basedir, homedir) if not os.path.exists(src_homedir): print(f"Directory {src_homedir} does not exist for user {user}, aborting") sys.exit(1) rsync_cmd = [ - 'rsync', '-av', - '--delete', '--ignore-errors', - src_homedir, dest_basedir + "rsync", + "-av", + "--delete", + "--ignore-errors", + src_homedir, + dest_basedir, ] if not dry_run: subprocess.check_output(rsync_cmd) return user, time.perf_counter() - start_time + def main(): argparser = argparse.ArgumentParser() - argparser.add_argument('hub_url', - help='Base URL of the JupyterHub to talk to' - ) - argparser.add_argument('hours_ago', - help='How recently should the user have been active to be rsynced', + argparser.add_argument("hub_url", help="Base URL of the JupyterHub to talk to") + argparser.add_argument( + "hours_ago", + help="How recently should the user have been active to be rsynced", type=int, ) - argparser.add_argument('src_basedir', - help='Base directory containing home directories to be rsynced' + argparser.add_argument( + "src_basedir", help="Base directory containing home directories to be rsynced" ) - argparser.add_argument('dest_basedir', - help='Base directory where home directories should be rsynced to' + argparser.add_argument( + "dest_basedir", + help="Base directory where home directories should be rsynced to", ) - argparser.add_argument('--actually-run-rsync', - action='store_true', - help="Actually run rsync, otherwise we just dry-run" + argparser.add_argument( + "--actually-run-rsync", + action="store_true", + help="Actually run rsync, otherwise we just dry-run", ) - argparser.add_argument('--concurrency', - type=int, - help='How many parallel rsyncs to run', - default=16 + argparser.add_argument( + "--concurrency", type=int, help="How many parallel rsyncs to run", default=16 ) args = argparser.parse_args() - if 'JUPYTERHUB_TOKEN' not in os.environ: - print('Could not find JUPYTERHUB_TOKEN in environment.') - print('Please get an admin user\'s token from {args.hub_url}/hub/token') + if "JUPYTERHUB_TOKEN" not in os.environ: + print("Could not find JUPYTERHUB_TOKEN in environment.") + print("Please get an admin user's token from {args.hub_url}/hub/token") sys.exit(1) time_since = datetime.now(timezone.utc) - timedelta(hours=args.hours_ago) users_since = [] - for user in get_all_users(args.hub_url, os.environ['JUPYTERHUB_TOKEN']): - if user['last_activity']: - if user['last_activity'] >= time_since: - users_since.append(user['name']) - + for user in get_all_users(args.hub_url, os.environ["JUPYTERHUB_TOKEN"]): + if user["last_activity"]: + if user["last_activity"] >= time_since: + users_since.append(user["name"]) pool = ThreadPoolExecutor(max_workers=args.concurrency) futures = [] @@ -166,20 +168,28 @@ def main(): # tarring is CPU bound, so we can parallelize trivially. # FIXME: This should be tuneable, or at least default to some multiple of number of cores on the system - future = pool.submit(rsync, - user, args.src_basedir, args.dest_basedir, not args.actually_run_rsync + future = pool.submit( + rsync, + user, + args.src_basedir, + args.dest_basedir, + not args.actually_run_rsync, ) futures.append(future) for future in as_completed(futures): completed_user, duration = future.result() completed_futures += 1 - print(f'Finished {completed_futures} of {len(users_since)} in {duration:0.3f} - user {completed_user}') + print( + f"Finished {completed_futures} of {len(users_since)} in {duration:0.3f} - user {completed_user}" + ) if not args.actually_run_rsync: print("No rsync commands were actually performed") - print("Check the rsync commands output, and then run this command with `--actually-run-rsync`") + print( + "Check the rsync commands output, and then run this command with `--actually-run-rsync`" + ) -if __name__ == '__main__': - main() +if __name__ == "__main__": + main() diff --git a/scripts/user-image-management/manage-image-repos.py b/scripts/user-image-management/manage-image-repos.py index 15890b2f2..1fad8a4ba 100755 --- a/scripts/user-image-management/manage-image-repos.py +++ b/scripts/user-image-management/manage-image-repos.py @@ -5,10 +5,13 @@ To use this tool, copy it to a directory in your PATH or run it directly from this directory. """ + import argparse import subprocess +import sys import os + def clone(args): """ Clone all repositories in the config file to the destination directory. @@ -36,20 +39,19 @@ def clone(args): continue print("Renaming origin to upstream...") subprocess.check_call( - ["git", "remote", "rename", "origin", "upstream"], - cwd=path + ["git", "remote", "rename", "origin", "upstream"], cwd=path ) origin = line.replace("berkeley-dsep-infra", args.github_user) print(f"Setting origin to {origin}...") subprocess.check_call( - ["git", "remote", "add", "origin", origin], - cwd=path + ["git", "remote", "add", "origin", origin], cwd=path ) subprocess.check_call(["git", "remote", "-v"], cwd=path) print() + def sync(args): """ Sync all repositories in the config file to the destination directory. @@ -69,7 +71,7 @@ def sync(args): print(f"Syncing {name} from {line} in {path}...") subprocess.check_call(["git", "switch", "main"], cwd=path) subprocess.check_call(["git", "fetch", "--all", "--prune"], cwd=path) - subprocess.check_call(["git", "rebase", f"upstream/main"], cwd=path) + subprocess.check_call(["git", "rebase", "upstream/main"], cwd=path) if args.push: if not args.origin: @@ -81,6 +83,7 @@ def sync(args): print() + def branch(args): """ Create a new feature branch in all repositories in the config file. @@ -99,6 +102,7 @@ def branch(args): subprocess.check_call(["git", "switch", "-c", args.branch], cwd=path) print() + def push(args): """ Push all repositories in the config file to a remote. @@ -125,6 +129,7 @@ def push(args): subprocess.check_call(["git", "push", remote, args.branch], cwd=path) print() + def main(args): if args.command == "clone": clone(args) @@ -135,6 +140,7 @@ def main(args): elif args.command == "push": push(args) + if __name__ == "__main__": argparser = argparse.ArgumentParser() subparsers = argparser.add_subparsers(dest="command") @@ -143,71 +149,51 @@ def main(args): "-c", "--config", default="repos.txt", - help="Path to file containing list of repositories to clone." + help="Path to file containing list of repositories to clone.", ) argparser.add_argument( - "-d", - "--destination", - default=".", - help="Location of the image repositories." + "-d", "--destination", default=".", help="Location of the image repositories." ) sync_parser = subparsers.add_parser( - "sync", - help="Sync all image repositories to the latest version." + "sync", help="Sync all image repositories to the latest version." ) sync_parser.add_argument( - "-p", - "--push", - action="store_true", - help="Push synced repo to a remote." + "-p", "--push", action="store_true", help="Push synced repo to a remote." ) sync_parser.add_argument( "-o", "--origin", default="origin", - help="Origin to push to. This is optional and defaults to 'origin'." + help="Origin to push to. This is optional and defaults to 'origin'.", ) - clone_parser = subparsers.add_parser( - "clone", - help="Clone all image repositories." - ) + clone_parser = subparsers.add_parser("clone", help="Clone all image repositories.") clone_parser.add_argument( "-s", "--set-origin", action="store_true", - help="Set the origin of the cloned repository to the user's GitHub." + help="Set the origin of the cloned repository to the user's GitHub.", ) clone_parser.add_argument( - "-g", - "--github-user", - help="GitHub user to set the origin to." + "-g", "--github-user", help="GitHub user to set the origin to." ) branch_parser = subparsers.add_parser( - "branch", - help="Create a new feature branch in all image repositories." + "branch", help="Create a new feature branch in all image repositories." ) branch_parser.add_argument( - "-b", - "--branch", - help="Name of the new feature branch to create." + "-b", "--branch", help="Name of the new feature branch to create." ) push_parser = subparsers.add_parser( - "push", - help="Push all image repositories to a remote." + "push", help="Push all image repositories to a remote." ) push_parser.add_argument( "-o", "--origin", - help="Origin to push to. This is optional and defaults to 'origin'." - ) - push_parser.add_argument( - "-b", - "--branch", - help="Name of the branch to push." + help="Origin to push to. This is optional and defaults to 'origin'.", ) + push_parser.add_argument("-b", "--branch", help="Name of the branch to push.") args = argparser.parse_args() main(args) diff --git a/vendor/google/ugr/gke/templates/gke_template.py b/vendor/google/ugr/gke/templates/gke_template.py index 4f8e96b84..fc0083248 100644 --- a/vendor/google/ugr/gke/templates/gke_template.py +++ b/vendor/google/ugr/gke/templates/gke_template.py @@ -4,75 +4,66 @@ def GenerateConfig(context): """Generates the YAML resource configuration.""" - project = context.env['project'] - cluster_name = context.properties['clusterName'] - region = context.properties['region'] - pool_name = context.properties['poolName'] - date_suffix = context.properties['dateSuffix'] # Format: yyyy-mm-dd - node_location = context.properties['nodeLocation'] - initial_node_count = context.properties['initialNodeCount'] - disk_size_gb = context.properties['diskSizeGb'] - machine_type = context.properties['machineType'] - min_node_count = context.properties['minNodeCount'] - max_node_count = context.properties['maxNodeCount'] + project = context.env["project"] + cluster_name = context.properties["clusterName"] + region = context.properties["region"] + pool_name = context.properties["poolName"] + date_suffix = context.properties["dateSuffix"] # Format: yyyy-mm-dd + node_location = context.properties["nodeLocation"] + initial_node_count = context.properties["initialNodeCount"] + disk_size_gb = context.properties["diskSizeGb"] + machine_type = context.properties["machineType"] + min_node_count = context.properties["minNodeCount"] + max_node_count = context.properties["maxNodeCount"] - resources = [{ - 'name': cluster_name, - 'type': 'gcp-types/container-v1:projects.locations.clusters', - 'properties': { - 'parent': f'projects/{project}/locations/{region}', - 'cluster': { - 'name': cluster_name, - 'initialClusterVersion': 'latest', - 'location': region, - 'locations': [node_location], - 'ipAllocationPolicy': { - 'useIpAliases': True + resources = [ + { + "name": cluster_name, + "type": "gcp-types/container-v1:projects.locations.clusters", + "properties": { + "parent": f"projects/{project}/locations/{region}", + "cluster": { + "name": cluster_name, + "initialClusterVersion": "latest", + "location": region, + "locations": [node_location], + "ipAllocationPolicy": {"useIpAliases": True}, + "addonsConfig": {"httpLoadBalancing": {"disabled": True}}, + "nodePools": [ + { + "name": f"{pool_name}-{date_suffix}", + "initialNodeCount": initial_node_count, + "config": { + "diskSizeGb": disk_size_gb, + "diskType": "pd-balanced", + "machineType": machine_type, + "imageType": "COS_CONTAINERD", + "labels": { + "hub.jupyter.org/pool-name": f"{pool_name}-pool-{date_suffix}" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + "https://www.googleapis.com/auth/servicecontrol", + "https://www.googleapis.com/auth/service.management.readonly", + "https://www.googleapis.com/auth/trace.append", + ], + }, + "autoscaling": { + "enabled": True, + "maxNodeCount": max_node_count, + "minNodeCount": min_node_count, + }, + "management": {"autoUpgrade": False, "autoRepair": True}, + "locations": [node_location], + } + ], + "networkPolicy": {"enabled": True}, }, - 'addonsConfig': { - 'httpLoadBalancing': { - 'disabled': True - } - }, - 'nodePools': [{ - 'name': f'{pool_name}-{date_suffix}', - 'initialNodeCount': initial_node_count, - 'config': { - 'diskSizeGb': disk_size_gb, - 'diskType': 'pd-balanced', - 'machineType': machine_type, - 'imageType': 'COS_CONTAINERD', - 'labels': { - 'hub.jupyter.org/pool-name': f'{pool_name}-pool-{date_suffix}' - }, - 'oauthScopes': [ - 'https://www.googleapis.com/auth/compute', - 'https://www.googleapis.com/auth/devstorage.read_only', - 'https://www.googleapis.com/auth/logging.write', - 'https://www.googleapis.com/auth/monitoring', - 'https://www.googleapis.com/auth/servicecontrol', - 'https://www.googleapis.com/auth/service.management.readonly', - 'https://www.googleapis.com/auth/trace.append' - ], - }, - 'autoscaling': { - 'enabled': True, - 'maxNodeCount': max_node_count, - 'minNodeCount': min_node_count - }, - 'management': { - 'autoUpgrade': False, - 'autoRepair': True - }, - 'locations': [ - node_location - ] - }], - 'networkPolicy': { - 'enabled': True - } - } + }, } - }] + ] - return {'resources': resources} + return {"resources": resources} diff --git a/vendor/google/ugr/gke/templates/node_pool_template.py b/vendor/google/ugr/gke/templates/node_pool_template.py index 19a9806a2..fa0d192de 100644 --- a/vendor/google/ugr/gke/templates/node_pool_template.py +++ b/vendor/google/ugr/gke/templates/node_pool_template.py @@ -4,50 +4,49 @@ def GenerateConfig(context): """Generates the YAML resource configuration for a node pool.""" - pool_name = context.properties['poolName'] - cluster_name = context.properties['clusterName'] - region = context.properties['region'] - date_suffix = context.properties['dateSuffix'] # Format: yyyy-mm-dd - initial_node_count = context.properties['initialNodeCount'] - disk_size_gb = context.properties['diskSizeGb'] - machine_type = context.properties['machineType'] - min_node_count = context.properties['minNodeCount'] - max_node_count = context.properties['maxNodeCount'] + pool_name = context.properties["poolName"] + cluster_name = context.properties["clusterName"] + region = context.properties["region"] + date_suffix = context.properties["dateSuffix"] # Format: yyyy-mm-dd + initial_node_count = context.properties["initialNodeCount"] + disk_size_gb = context.properties["diskSizeGb"] + machine_type = context.properties["machineType"] + min_node_count = context.properties["minNodeCount"] + max_node_count = context.properties["maxNodeCount"] - resources = [{ - 'name': f'user-{pool_name}-{date_suffix}', - 'type': 'gcp-types/container-v1:projects.locations.clusters.nodePools', - 'properties': { - 'parent': f'projects/{context.env["project"]}/locations/{region}/clusters/{cluster_name}', - 'nodePool': { - 'name': f'{pool_name}-pool', - 'initialNodeCount': initial_node_count, - 'config': { - 'machineType': machine_type, - 'diskSizeGb': disk_size_gb, - 'diskType': 'pd-balanced', - 'imageType': 'COS_CONTAINERD', - 'labels': { - 'hub.jupyter.org/pool-name': f'{pool_name}-pool' + resources = [ + { + "name": f"user-{pool_name}-{date_suffix}", + "type": "gcp-types/container-v1:projects.locations.clusters.nodePools", + "properties": { + "parent": f'projects/{context.env["project"]}/locations/{region}/clusters/{cluster_name}', + "nodePool": { + "name": f"{pool_name}-pool", + "initialNodeCount": initial_node_count, + "config": { + "machineType": machine_type, + "diskSizeGb": disk_size_gb, + "diskType": "pd-balanced", + "imageType": "COS_CONTAINERD", + "labels": {"hub.jupyter.org/pool-name": f"{pool_name}-pool"}, + "taints": [ + { + "key": "hub.jupyter.org_dedicated", + "value": "user", + "effect": "NO_SCHEDULE", + } + ], + "tags": ["hub-cluster"], }, - 'taints': [{ - 'key': 'hub.jupyter.org_dedicated', - 'value': 'user', - 'effect': 'NO_SCHEDULE' - }], - 'tags': ['hub-cluster'], - }, - 'autoscaling': { - 'enabled': True, - 'maxNodeCount': max_node_count, - 'minNodeCount': min_node_count - }, - 'management': { - 'autoUpgrade': False, - 'autoRepair': True + "autoscaling": { + "enabled": True, + "maxNodeCount": max_node_count, + "minNodeCount": min_node_count, + }, + "management": {"autoUpgrade": False, "autoRepair": True}, }, - } + }, } - }] + ] - return {'resources': resources} + return {"resources": resources}