From 18dcc5e3f7389f5a381c7e657a4fdfbe05105bfc Mon Sep 17 00:00:00 2001 From: kaarthik108 Date: Wed, 23 Aug 2023 15:50:39 +1200 Subject: [PATCH] Fix streamlit new component and cli refractor --- .github/ISSUE_TEMPLATE/bugs.yml | 64 ++++++ .github/ISSUE_TEMPLATE/feature-request.yml | 47 ++++ README.md | 2 +- pyproject.toml | 10 +- snowdev/cli/__init__.py | 0 snowdev/cli/commands.py | 127 +++++++++++ snowdev/cli/main.py | 9 + snowdev/{main.py => deployment.py} | 230 +++++--------------- snowdev/fillers/sproc/fill.toml | 4 +- snowdev/fillers/udf/fill.toml | 4 +- snowdev/functions/bot.py | 10 +- snowdev/functions/helper.py | 26 +-- snowdev/functions/utils/snowpark_methods.py | 17 +- 13 files changed, 348 insertions(+), 202 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bugs.yml create mode 100644 .github/ISSUE_TEMPLATE/feature-request.yml create mode 100644 snowdev/cli/__init__.py create mode 100644 snowdev/cli/commands.py create mode 100644 snowdev/cli/main.py rename snowdev/{main.py => deployment.py} (61%) diff --git a/.github/ISSUE_TEMPLATE/bugs.yml b/.github/ISSUE_TEMPLATE/bugs.yml new file mode 100644 index 0000000..e170014 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bugs.yml @@ -0,0 +1,64 @@ +name: Bug Report +description: Report a bug in the project +labels: [bug] + +body: + - type: markdown + attributes: + value: | + ## Bug Report + + Thank you for reporting a bug. Please provide as much information as possible. + + - type: input + id: python-version + attributes: + label: Python Version + description: Are you python 3.10 and above? + placeholder: e.g., 3.10.2 + validations: + required: true + + - type: input + id: snowpark-version + attributes: + label: Snowflake Snowpark Python Version + description: Are you using snowflake-snowpark-python 1.5.1? + placeholder: e.g., 1.5.1 + validations: + required: true + + - type: textarea + id: steps-to-reproduce + attributes: + label: Steps to reproduce + description: Provide a step-by-step description of the bug. + placeholder: Describe the steps to reproduce the bug. + validations: + required: true + + - type: textarea + id: expected-behavior + attributes: + label: Expected behavior + description: Describe what you expected to happen. + placeholder: Describe the expected behavior. + validations: + required: true + + - type: textarea + id: actual-behavior + attributes: + label: Actual behavior + description: Describe what actually happened. + placeholder: Describe the actual behavior. + validations: + required: true + + - type: textarea + id: additional-info + attributes: + label: Additional information + description: Provide any additional information or context. + placeholder: Any other details? + diff --git a/.github/ISSUE_TEMPLATE/feature-request.yml b/.github/ISSUE_TEMPLATE/feature-request.yml new file mode 100644 index 0000000..9399504 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yml @@ -0,0 +1,47 @@ +name: Feature Request +description: Suggest a new feature or enhancement for the project +labels: [enhancement] + +body: + - type: markdown + attributes: + value: | + ## Feature Request + + Thank you for suggesting a feature. Please provide as much information as possible. + + - type: input + id: title + attributes: + label: Feature Title + description: A short title for the feature. + placeholder: e.g., New data visualization tool + validations: + required: true + + - type: textarea + id: feature-description + attributes: + label: Feature Description + description: Describe the feature in detail. + placeholder: Describe the feature you'd like to see. + validations: + required: true + + - type: checkboxes + id: compatibility + attributes: + label: Compatibility + description: Which versions should this feature be compatible with? + options: + - label: Python 3.10 and above + required: true + - label: snowflake-snowpark-python 1.5.1 + required: true + + - type: textarea + id: additional-info + attributes: + label: Additional information + description: Provide any additional information or context. + placeholder: Any other details? diff --git a/README.md b/README.md index 9ce8d11..87fb0f6 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SnowDev - Snowpark Devops -[![Documentation](https://img.shields.io/badge/documentation-view-blue)](docs/quickstart.md) ![PyPI Downloads](https://img.shields.io/pypi/dm/snowdev) +[![Documentation](https://img.shields.io/badge/documentation-view-blue)](docs/quickstart.md) [![Downloads](https://static.pepy.tech/badge/snowdev)](https://pepy.tech/project/snowdev) SnowDev is a command-line utility designed for deploying various components related to Snowflake such as UDFs, stored procedures, and Streamlit applications using **Snowpark**. This tool streamlines tasks like initializing directories, local testing, uploading, and auto create components code using AI. diff --git a/pyproject.toml b/pyproject.toml index 5221cd8..748c620 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "snowdev" -version = "0.1.10" +version = "0.1.11" description = "snowdev: DevOps toolkit for Snowflake, facilitating seamless deployment of UDFs, stored procedures, and Streamlit apps using Snowpark's capabilities right from your local environment." authors = ["kaarthik "] readme = "README.md" @@ -21,15 +21,15 @@ snowflake-ml-python = "1.0.5" pyyaml = "^6.0.1" toml = "^0.10.2" openai = "^0.27.8" -langchain = "^0.0.265" +langchain = "0.0.265" +tiktoken = "0.4.0" +unstructured = "0.9.3" [tool.poetry.dev-dependencies] black = "^22.10.0" -tiktoken = "0.4.0" -unstructured = "^0.9.3" [tool.poetry.scripts] -snowdev = "snowdev.main:main" +snowdev = "snowdev.cli.main:main" [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/snowdev/cli/__init__.py b/snowdev/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/snowdev/cli/commands.py b/snowdev/cli/commands.py new file mode 100644 index 0000000..4e2283f --- /dev/null +++ b/snowdev/cli/commands.py @@ -0,0 +1,127 @@ +import click +from termcolor import colored + +from snowdev import SnowBot, SnowHelper +from snowdev.deployment import DeploymentArguments, DeploymentManager + + +@click.group() +@click.pass_context +def cli(ctx): + """Deploy Snowflake UDFs, Stored Procedures and Streamlit apps.""" + ctx.ensure_object(dict) + + +@cli.command() +def init(): + """Initialize the project structure.""" + DeploymentManager.create_directory_structure() + + +@cli.command() +@click.option("--udf", type=str, help="The name of the udf.") +@click.option("--sproc", type=str, help="The name of the stored procedure.") +@click.option("--streamlit", type=str, help="The name of the streamlit app.") +def new(udf, sproc, streamlit): + """Create a new component.""" + SnowHelper.create_new_component(udf, sproc, streamlit) + + +@cli.command() +@click.option("--udf", type=str, help="The name of the udf.") +@click.option("--sproc", type=str, help="The name of the stored procedure.") +def test(udf, sproc): + """Test the deployment.""" + deployment_args = DeploymentArguments(udf=udf, sproc=sproc) + manager = DeploymentManager(deployment_args) + manager.test_locally() + + +@cli.command() +def upload(): + """Upload static content.""" + manager = DeploymentManager() + manager.upload_static() + + +@cli.command() +@click.option("--package", type=str, help="Name of the package to zip and upload.") +def add(package): + """Add a package and optionally upload.""" + manager = DeploymentManager() + user_response = input( + colored("🤔 Do you want to upload the zip to stage? (yes/no): ", "cyan") + ) + + if user_response.lower() in ["yes", "y"]: + manager.deploy_package(package, upload=True) + else: + manager.deploy_package(package, upload=False) + + +@cli.command() +@click.option("--udf", type=str, help="The name of the udf.") +@click.option("--sproc", type=str, help="The name of the stored procedure.") +@click.option("--streamlit", type=str, help="The name of the streamlit app.") +@click.option("--embed", is_flag=True, help="Run the embeddings.") +def ai(udf, sproc, streamlit, embed): + """AI commands.""" + + if embed: + print(colored("Initializing AI...\n", "cyan")) + SnowBot.ai_embed() + return + + component_type, prompt = None, None + + if udf: + component_type = "udf" + prompt = udf + elif sproc: + component_type = "sproc" + prompt = sproc + elif streamlit: + component_type = "streamlit" + prompt = streamlit + + if not component_type: + print( + colored( + "⚠️ Please specify a type (--udf, --sproc, or --streamlit) along with the ai command.", + "yellow", + ) + ) + return + + component_name = input( + colored(f"🤔 Enter the {component_type.upper()} name: ", "cyan") + ) + + if SnowBot.component_exists(component_name, component_type): + print( + colored( + f"⚠️ Component named {component_name} already exists! Choose another name or check your directories.", + "yellow", + ) + ) + return + + SnowBot.create_new_ai_component( + component_name, prompt, template_type=component_type + ) + + +@cli.command() +@click.option("--sproc", type=str, help="The name of the stored procedure.") +@click.option("--udf", type=str, help="The name of the udf.") +@click.option("--streamlit", type=str, help="The name of the streamlit app.") +def deploy(sproc, udf, streamlit): + """Deploy components.""" + arguments = {"sproc": sproc, "udf": udf, "streamlit": streamlit} + args = DeploymentArguments(**arguments) + manager = DeploymentManager(args) + manager.main() + + +if __name__ == "__main__": + cli() diff --git a/snowdev/cli/main.py b/snowdev/cli/main.py new file mode 100644 index 0000000..c8dc9d1 --- /dev/null +++ b/snowdev/cli/main.py @@ -0,0 +1,9 @@ +from .commands import cli + + +def main(): + cli() + + +if __name__ == "__main__": + main() diff --git a/snowdev/main.py b/snowdev/deployment.py similarity index 61% rename from snowdev/main.py rename to snowdev/deployment.py index 5f56e22..bf14be5 100644 --- a/snowdev/main.py +++ b/snowdev/deployment.py @@ -1,10 +1,9 @@ -import argparse import os import subprocess from typing import Optional import toml -from pydantic import BaseModel +from pydantic import BaseModel, validator from termcolor import colored from snowdev import ( @@ -13,7 +12,6 @@ SnowHelper, SnowPackageZip, StreamlitAppDeployer, - SnowBot, ) @@ -25,13 +23,32 @@ class DeploymentArguments(BaseModel): upload: Optional[str] package: Optional[str] + @validator("udf", "sproc", "streamlit", pre=True, always=True) + def path_exists(cls, value, values, field, **kwargs): + if value: + path = "" + if "udf" in field.name: + path = os.path.join(DeploymentManager.UDF_PATH, value) + elif "sproc" in field.name: + path = os.path.join(DeploymentManager.SPROC_PATH, value) + elif "streamlit" in field.name: + path = os.path.join(DeploymentManager.STREAMLIT_PATH, value) + + if not os.path.exists(path): + error_message = colored( + f"\n❌ ERROR: The specified path '{path}' does not exist. Please ensure the path is correct and try again.\n", + "red", + ) + raise ValueError(error_message) + return value + class DeploymentManager: UDF_PATH = "src/udf/" SPROC_PATH = "src/sproc/" STREAMLIT_PATH = "src/streamlit/" - def __init__(self, args): + def __init__(self, args=None): self.args = args self.stage_name = "SNOWDEV" self.session = SnowflakeConnection().get_session() @@ -115,7 +132,7 @@ def deploy_function(self, filepath, is_sproc): ) return - print("packages are----", packages) + # print("packages are----", packages) try: self.snow_deploy = SnowflakeRegister(session=self.session) self.snow_deploy.main( @@ -254,175 +271,48 @@ def deploy_task(self, taskname): def deploy_pipe(self, pipe_name): pass - -def create_directory_structure(): - dirs_to_create = { - "src": ["sproc", "streamlit", "udf"], - "static": ["packages"], - } - - # Initialize a flag to check if the structure already exists - structure_already_exists = True - - for root_dir, sub_dirs in dirs_to_create.items(): - if not os.path.exists(root_dir): - structure_already_exists = False - os.mkdir(root_dir) - for sub_dir in sub_dirs: - os.mkdir(os.path.join(root_dir, sub_dir)) - - files_to_create = [".env", ".gitignore", "pyproject.toml"] - - if not os.path.exists(".env"): - structure_already_exists = False - with open(".env", "w") as f: - f.write("") - - if not os.path.exists(".gitignore"): - structure_already_exists = False - with open(".gitignore", "w") as f: - f.write("*.pyc\n__pycache__/\n.env") - - if not os.path.exists("pyproject.toml"): - structure_already_exists = False - template_path = SnowHelper.get_template_path("fillers/pyproject.toml") - try: - with open(template_path, "r") as template_file: - content = template_file.read() - - with open("pyproject.toml", "w") as f: - f.write(content) - except FileNotFoundError: - print(colored(f"Error: Template {template_path} not found!", "red")) - - if structure_already_exists: - print(colored("Project structure is already initialized!", "yellow")) - else: - print(colored("Project structure initialized!", "green")) - - - -def parse_args(): - parser = argparse.ArgumentParser( - description="Deploy Snowflake UDFs, Stored Procedures and Streamlit apps." - ) - - parser.add_argument( - "command", - choices=["init", "test", "deploy", "upload", "add", "new", "ai"], - help="The main command to execute.", - ) - parser.add_argument( - "--udf", type=str, help="The name of the udf." - ) - parser.add_argument( - "--sproc", - type=str, - help="The name of the stored procedure.", - ) - parser.add_argument( - "--streamlit", - type=str, - help="The name of the streamlit app.", - ) - - parser.add_argument( - "--upload", - type=str, - choices=[ - "static" - ], # expand this list if you have other things to upload in the future - help="Specify what to upload (e.g., static).", - ) - parser.add_argument( - "--package", - type=str, - help="Name of the package to zip and upload to the static folder.", - ) - parser.add_argument( - "--embed", - action="store_true", - help="Run the embeddings", - ) - - return parser.parse_args() - - -def execute_command(args): - if args.command == "init": - create_directory_structure() - return - elif args.command == "test": - deployment_manager = DeploymentManager(args) - deployment_manager.test_locally() - return - elif args.command == "upload": - deployment_manager = DeploymentManager(args) - deployment_manager.upload_static() - return - elif args.command == "add": - deployment_manager = DeploymentManager(args) - user_response = input( - colored("🤔 Do you want to upload the zip to stage? (yes/no): ", "cyan") - ) - if user_response.lower() in ["yes", "y"]: - deployment_manager.deploy_package(args.package, upload=True) - return - deployment_manager.deploy_package(args.package, upload=False) - return - - elif args.command == "ai": - if args.embed: - print("Initializing AI...") - SnowBot.ai_embed() - return - - component_details = { - k: v - for k, v in vars(args).items() - if k in ["udf", "sproc", "streamlit"] and v + @staticmethod + def create_directory_structure(): + dirs_to_create = { + "src": ["sproc", "streamlit", "udf"], + "static": ["packages"], } - if not component_details: - print( - colored( - "⚠️ Please specify a type (--udf, --sproc, or --streamlit) along with the ai command.", - "yellow", - ) - ) - return - - component_type, prompt = list(component_details.items())[0] + # Initialize a flag to check if the structure already exists + structure_already_exists = True - component_name = input( - colored(f"🤔 Enter the {component_type.upper()} name: ", "cyan") - ) + for root_dir, sub_dirs in dirs_to_create.items(): + if not os.path.exists(root_dir): + structure_already_exists = False + os.mkdir(root_dir) + for sub_dir in sub_dirs: + os.mkdir(os.path.join(root_dir, sub_dir)) - if SnowBot.component_exists(component_name, component_type): - print( - colored( - f"⚠️ Component named {component_name} already exists! Choose another name or check your directories.", - "yellow", - ) - ) - return - - SnowBot.create_new_ai_component( - component_name, prompt, template_type=component_type - ) - return - - elif args.command == "new": - SnowHelper.create_new_component(vars(args)) - return + files_to_create = [".env", ".gitignore", "pyproject.toml"] - elif args.command == "deploy": - deployment_args = DeploymentArguments(**vars(args)) - deployment_manager = DeploymentManager(deployment_args) - deployment_manager.main() - return + if not os.path.exists(".env"): + structure_already_exists = False + with open(".env", "w") as f: + f.write("") + if not os.path.exists(".gitignore"): + structure_already_exists = False + with open(".gitignore", "w") as f: + f.write("*.pyc\n__pycache__/\n.env") -def main(): - args = parse_args() - execute_command(args) + if not os.path.exists("pyproject.toml"): + structure_already_exists = False + template_path = SnowHelper.get_template_path("fillers/pyproject.toml") + try: + with open(template_path, "r") as template_file: + content = template_file.read() + + with open("pyproject.toml", "w") as f: + f.write(content) + except FileNotFoundError: + print(colored(f"Error: Template {template_path} not found!", "red")) + + if structure_already_exists: + print(colored("Project structure is already initialized!", "yellow")) + else: + print(colored("Project structure initialized!", "green")) diff --git a/snowdev/fillers/sproc/fill.toml b/snowdev/fillers/sproc/fill.toml index 5e7d71c..a957e1e 100644 --- a/snowdev/fillers/sproc/fill.toml +++ b/snowdev/fillers/sproc/fill.toml @@ -1,3 +1,3 @@ [tool.poetry.dependencies] -python = "3.9.17" -snowflake-snowpark-python = "1.5.0" \ No newline at end of file +python = "3.10.0" +snowflake-snowpark-python = "1.5.1" \ No newline at end of file diff --git a/snowdev/fillers/udf/fill.toml b/snowdev/fillers/udf/fill.toml index 8b2c2be..f3a61b6 100644 --- a/snowdev/fillers/udf/fill.toml +++ b/snowdev/fillers/udf/fill.toml @@ -1,4 +1,4 @@ [tool.poetry.dependencies] -python = "3.9.17" -snowflake-snowpark-python = "1.5.0" +python = "3.10.0" +snowflake-snowpark-python = "1.5.1" pandas = "2.0.3" \ No newline at end of file diff --git a/snowdev/functions/bot.py b/snowdev/functions/bot.py index 0a7f936..c075808 100644 --- a/snowdev/functions/bot.py +++ b/snowdev/functions/bot.py @@ -19,6 +19,7 @@ MODEL = "gpt-4" + class SnowBot: TEMPLATES = { @@ -135,9 +136,14 @@ def create_new_ai_component(component_name, prompt, template_type): if matches: # Take the first match, as there might be multiple code blocks - response_content = matches[0].strip() + response_content = matches[0].strip() else: - print(colored("Unexpected response content format. Expected code block not found. Please try again", "red")) + print( + colored( + "Unexpected response content format. Expected code block not found. Please try again", + "red", + ) + ) component_folder = os.path.join("src", template_type, component_name) os.makedirs(component_folder, exist_ok=True) diff --git a/snowdev/functions/helper.py b/snowdev/functions/helper.py index 650d84f..f70c675 100644 --- a/snowdev/functions/helper.py +++ b/snowdev/functions/helper.py @@ -156,16 +156,14 @@ def create_new_component(cls, args_dict): ("py", "streamlit_app.py"), ("yml", "environment.yml"), ]: - template_name = "fillers/streamlit/fill." + template_ext - template_path = os.path.join("snowdev", template_name) - - if os.path.exists(template_path): - with open(template_path, "r") as template_file: - content = template_file.read() + try: + template_content = pkg_resources.resource_string( + "snowdev", f"fillers/streamlit/fill.{template_ext}" + ).decode("utf-8") with open(os.path.join(new_item_path, output_name), "w") as f: - f.write(content) - else: + f.write(template_content) + except FileNotFoundError: print( colored( f"No template found for {item_type} with extension {template_ext}. Creating an empty {output_name}...", @@ -176,15 +174,15 @@ def create_new_component(cls, args_dict): pass else: for ext, template_name in cls.TEMPLATES[item_type].items(): - template_path = os.path.join("snowdev", template_name) - if os.path.exists(template_path): - with open(template_path, "r") as template_file: - content = template_file.read() + try: + template_content = pkg_resources.resource_string( + "snowdev", template_name + ).decode("utf-8") filename = "app.py" if ext == "py" else "app.toml" with open(os.path.join(new_item_path, filename), "w") as f: - f.write(content) - else: + f.write(template_content) + except FileNotFoundError: filename = "app.py" if ext == "py" else "app.toml" print( colored( diff --git a/snowdev/functions/utils/snowpark_methods.py b/snowdev/functions/utils/snowpark_methods.py index 8c89b3f..f81a6d8 100644 --- a/snowdev/functions/utils/snowpark_methods.py +++ b/snowdev/functions/utils/snowpark_methods.py @@ -3,8 +3,8 @@ import pkg_resources from snowflake import snowpark, ml -class SnowparkMethods: +class SnowparkMethods: @classmethod def _extract_methods_and_docs_from_session(cls, module): """ @@ -13,7 +13,7 @@ def _extract_methods_and_docs_from_session(cls, module): items = [] # Check if the module has a Session class - SessionClass = getattr(module, 'Session', None) + SessionClass = getattr(module, "Session", None) if SessionClass: for name, obj in inspect.getmembers(SessionClass): if inspect.isfunction(obj) or inspect.ismethod(obj): @@ -26,9 +26,11 @@ def _write_to_markdown_file(cls, filename, data): """ Write method names and docstrings to a markdown file. """ - with open(filename, 'w') as f: + with open(filename, "w") as f: f.write("# Snowflake Session Methods\n") - f.write("This document lists methods available in the Snowflake Session class, along with their documentation.\n\n") + f.write( + "This document lists methods available in the Snowflake Session class, along with their documentation.\n\n" + ) for name, doc in data: f.write(f"## {name}\n\n") f.write(f"{doc}\n\n") @@ -41,12 +43,15 @@ def generate_documentation(cls): all_items = snowpark_items + ml_items # Using pkg_resources to get the path inside the package - knowledge_dir = pkg_resources.resource_filename('snowdev.functions.utils', 'knowledge') - filepath = os.path.join(knowledge_dir, 'documentation.md') + knowledge_dir = pkg_resources.resource_filename( + "snowdev.functions.utils", "knowledge" + ) + filepath = os.path.join(knowledge_dir, "documentation.md") cls._write_to_markdown_file(filepath, all_items) print(f"Documentation written to '{filepath}'") + if __name__ == "__main__": SnowparkMethods.generate_documentation()