From 38ad28695769f8fab22baf1cf7d712fbb19f5158 Mon Sep 17 00:00:00 2001 From: Sebastian Quintero Date: Fri, 2 Feb 2024 13:22:37 -0500 Subject: [PATCH] Update project configuration and add new files --- .gitignore | 3 + .golangci.yml | 8 +- .nextmv/README.md | 23 +++ requirements.txt => .nextmv/requirements.txt | 0 .nextmv/update_apps.py | 195 ++++++++++++++++++ .../workflow-configuration.yml | 0 VERSION | 1 - scripts/update_app.sh | 55 ----- scripts/update_manifest.py | 104 ---------- 9 files changed, 225 insertions(+), 164 deletions(-) create mode 100644 .nextmv/README.md rename requirements.txt => .nextmv/requirements.txt (100%) create mode 100644 .nextmv/update_apps.py rename workflow-configuration.yml => .nextmv/workflow-configuration.yml (100%) delete mode 100644 VERSION delete mode 100644 scripts/update_app.sh delete mode 100644 scripts/update_manifest.py diff --git a/.gitignore b/.gitignore index 9d8009b..7ff4e26 100644 --- a/.gitignore +++ b/.gitignore @@ -162,3 +162,6 @@ cython_debug/ # Go stuff go.work go.work.sum + +# VSCode stuff +.vscode/ diff --git a/.golangci.yml b/.golangci.yml index af1f52e..ccd9e5f 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ # Options for analysis running. Details: https://golangci-lint.run/usage/configuration/#run-configuration run: timeout: 5m - go: "1.21" + go: "1.19" # Configures linters. Details: https://golangci-lint.run/usage/linters linters-settings: @@ -17,11 +17,11 @@ linters-settings: min-complexity: 45 # Set correct go version. gosimple: - go: "1.21" + go: "1.19" staticcheck: - go: "1.21" + go: "1.19" stylecheck: - go: "1.21" + go: "1.19" # Check case of struct tags tagliatelle: case: diff --git a/.nextmv/README.md b/.nextmv/README.md new file mode 100644 index 0000000..091b644 --- /dev/null +++ b/.nextmv/README.md @@ -0,0 +1,23 @@ +# Nextmv Workflows + +This directory contains functionality for managing community apps. + +## Requirements + +- For Python scripts please install the packages in `requirements.txt`: + + ```bash + pip install -r requirements.txt + ``` + +## Use + +- To update one or more apps. Standing in the `./nextmv` directory, run: + + ```bash + python update_apps.py \ + -a app1=version1,app2=version2 \ + -b BUCKET \ + -f FOLDER \ + -m MANIFEST_FILE + ``` diff --git a/requirements.txt b/.nextmv/requirements.txt similarity index 100% rename from requirements.txt rename to .nextmv/requirements.txt diff --git a/.nextmv/update_apps.py b/.nextmv/update_apps.py new file mode 100644 index 0000000..38592b7 --- /dev/null +++ b/.nextmv/update_apps.py @@ -0,0 +1,195 @@ +""" +This script updates one or more applications by: + +1. Updating the version. +2. Updating the SDK version (if the app is written in Go). +3. Updating the manifest file and uploading it. + +Execute from the ./nextmv directory: + +```bash +python update_apps.py \ + -a app1=version1,app2=version2 \ + -b BUCKET \ + -f FOLDER \ + -m MANIFEST_FILE +``` +""" + +import argparse +import copy +import os +import subprocess +from typing import Any + +import yaml +from boto3 import client as s3Client + +parser = argparse.ArgumentParser(description="Update community apps in the Marketplace.") +parser.add_argument( + "--apps", + "-a", + type=str, + help="Apps to release, with the version. Example: knapsack-gosdk=v1.0.0,knapsack-pyomo=v1.0.0", + required=True, +) +parser.add_argument( + "--bucket", + "-b", + type=str, + help="S3 bucket.", + required=True, +) +parser.add_argument( + "--folder", + "-f", + type=str, + help="S3 bucket folder.", + required=True, +) +parser.add_argument( + "--manifest", + "-m", + type=str, + help="Manifest file.", + required=True, +) +args = parser.parse_args() + + +def main(): + """ + Entry point for the script. + """ + + apps = [ + { + "name": app.split("=")[0], + "version": app.split("=")[1], + } + for app in args.apps.split(",") + ] + apps.sort(key=lambda x: x["name"]) + client = s3Client("s3") + + manifest = get_manifest( + client=client, + bucket=args.bucket, + folder=args.folder, + manifest_file=args.manifest, + ) + workflow_configuration = read_yaml( + filepath=os.path.join(os.getcwd(), "workflow-configuration.yml"), + ) + for app in apps: + if "v" not in app["version"]: + app["version"] = f"v{app['version']}" + + update_app( + name=app["name"], + version=app["version"], + workflow_configuration=workflow_configuration, + ) + manifest = update_manifest(manifest, app) + + upload_manifest( + client=client, + bucket=args.bucket, + folder=args.folder, + manifest_file=args.manifest, + manifest=manifest, + ) + + +def update_app(name: str, version: str, workflow_configuration: dict[str, Any]): + """Updates the app with the new version.""" + + workflow_info = {} + for app in workflow_configuration["apps"]: + if app["name"] == name: + workflow_info = app + break + + with open(os.path.join(os.getcwd(), "..", name, "VERSION"), "w") as f: + f.write(version + "\n") + + if workflow_info["type"] == "go": + sdk_version = workflow_info["sdk_version"] + if "v" not in sdk_version: + sdk_version = f"v{sdk_version}" + + _ = subprocess.run( + ["go", "get", f"github.com/nextmv-io/sdk@{sdk_version}"], + capture_output=True, + text=True, + check=True, + cwd=os.path.join("..", name), + ) + _ = subprocess.run( + ["go", "mod", "tidy"], + capture_output=True, + text=True, + check=True, + cwd=os.path.join("..", name), + ) + + +def get_manifest( + client: s3Client, + bucket: str, + folder: str, + manifest_file: str, +) -> dict[str, Any]: + """Returns the manifest from the S3 bucket.""" + + result = client.get_object(Bucket=bucket, Key=f"{folder}/{manifest_file}") + return yaml.safe_load(result["Body"].read()) + + +def upload_manifest( + client: s3Client, + bucket: str, + folder: str, + manifest_file: str, + manifest: dict[str, Any], +): + """Uploads the manifest to the S3 bucket.""" + + class Dumper(yaml.Dumper): + """Custom YAML dumper that does not use the default flow style.""" + + def increase_indent(self, flow=False, indentless=False): + return super().increase_indent(flow, False) + + client.put_object( + Bucket=bucket, + Key=f"{folder}/{manifest_file}", + Body=yaml.dump(manifest, Dumper=Dumper, default_flow_style=False), + ) + + +def update_manifest( + old: dict[str, Any], + app: dict[str, Any], +) -> dict[str, Any]: + """Updates the manifest with the new apps.""" + + new = copy.deepcopy(old) + for manifest_app in new["apps"]: + if manifest_app["name"] == app["name"]: + manifest_app["latest"] = app["version"] + manifest_app["versions"].append(app["version"]) + break + + return new + + +def read_yaml(filepath: str) -> dict[str, Any]: + """Returns the YAML file in the path.""" + + with open(filepath) as f: + return yaml.safe_load(f) + + +if __name__ == "__main__": + main() diff --git a/workflow-configuration.yml b/.nextmv/workflow-configuration.yml similarity index 100% rename from workflow-configuration.yml rename to .nextmv/workflow-configuration.yml diff --git a/VERSION b/VERSION deleted file mode 100644 index b82608c..0000000 --- a/VERSION +++ /dev/null @@ -1 +0,0 @@ -v0.1.0 diff --git a/scripts/update_app.sh b/scripts/update_app.sh deleted file mode 100644 index b16964e..0000000 --- a/scripts/update_app.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/bash -set -e - -# ================= README ================= -# Updates an app to the desired version. - -# Required exports. -# APP: the name of the app to update. -# VERSION: the version to update to. - -# ================= Checks ================= -# Checks that APP is set. -echo "🐰 Checking the value of APP" -if [[ -z "${APP}" ]]; then - echo "❌ APP must be set" - exit -1 -fi - -# Checks that VERSION is set. -echo "🐰 Checking the value of VERSION" -if [[ -z "${VERSION}" ]]; then - echo "❌ VERSION must be set" - exit -1 -fi - -# ================= App info ================= -# Get the info of the app from the configuration file. -WORFKLOW_CONFIGURATION="workflow-configuration.yml" -INFO=$(yq '.apps[] | select(.name == strenv(APP))' workflow-configuration.yml) -echo "🐰 App info:" -echo "$INFO" | yq . - -# ================= Directory ================= -# Change to the APP directory. -cd "$(dirname "$0")/.." -echo "🐰 Current dir is: $(pwd)" -echo "🐰 Changing to dir $APP" -cd $APP - -# ================= Update ================= -# Update the VERSION file to the VERSION. -OLD=$(cat VERSION) -echo "🐰 Updating the VERSION file from $OLD to $VERSION" -echo $VERSION > VERSION - -# If the app is a Go app, update the go.mod file. -TYPE=$(echo "$INFO" | yq .type) -echo "🐰 App type is $TYPE" -if [[ $TYPE == "go" ]]; then - SDK_VERSION=$(echo "$INFO" | yq .sdk_version) - echo "🐰 SDK version is $SDK_VERSION" - echo "🐰 Updating the go.mod file" - go get github.com/nextmv-io/sdk@$SDK_VERSION - go mod tidy -fi diff --git a/scripts/update_manifest.py b/scripts/update_manifest.py deleted file mode 100644 index 6c11122..0000000 --- a/scripts/update_manifest.py +++ /dev/null @@ -1,104 +0,0 @@ -""" -This script updates the manifest file with the latest version of the release -and the apps. It assumes that there is a `manifest.yml` file which corresponds -to the old manifest. It writes the new manifest to `new_manifest.yml`. Versions -are taken from the `VERSION` file in each app directory, and the root. - -Execute from the root: - -```bash -python scripts/update_manifest.py -``` -""" -import copy -import os -from typing import Any - -import yaml - - -def main(): - """ - Entry point for the script. - """ - - apps = [ - { - "name": app["name"], - "type": app["type"], - "version": version(dir=app["name"]), - } - for app in open_yaml_file(filepath="workflow-configuration.yml")["apps"] - ] - apps.sort(key=lambda x: x["name"]) - old = open_yaml_file(filepath="manifest.yml") - new = update_manifest(old=old, apps=apps) - with open("new_manifest.yml", "w") as f: - yaml.dump(new, f) - - -def update_manifest( - old: dict[str, Any], - apps: list[dict[str, str]], -) -> dict[str, Any]: - """ - Updates the manifest with the new apps. - - Args: - old: The old manifest. - apps: The list of apps. - - Returns: - The new manifest. - """ - - new = copy.deepcopy(old) - new_version = version(dir=".") - new["latest"] = new_version - release = { - "version": new_version, - "apps": apps, - } - new["releases"].append(release) - - return new - - -def open_yaml_file(filepath: str) -> dict[str, Any]: - """ - Returns the YAML file in the path. - - Args: - filepath: The path of the file. - - Returns: - The content of the YAML file. - - Raises: - FileNotFoundError: If the file is not found. - """ - - with open(filepath) as f: - return yaml.safe_load(f) - - -def version(dir: str) -> str: - """ - Returns the version in the given directory. - - Args: - dir: The directory. - - Returns: - The version. - - Raises: - FileNotFoundError: If the VERSION file is not found. - """ - - with open(os.path.join(dir, "VERSION")) as f: - return f.read().strip() - - -if __name__ == "__main__": - main()