diff --git a/models/models.py b/models/models.py index b4ab723..a7c2271 100644 --- a/models/models.py +++ b/models/models.py @@ -58,10 +58,22 @@ class NumberSlider(AbstractComponentWithId): Name: Literal["Number Slider"] Value: str = Field( "0..2..4", - alias='Value', - description="The range of values for the Number Slider. " - "In the format '....'. " - "Give a decent range of values to allow for flexibility" + description=textwrap.dedent( + """ + The range of values for the Number Slider + In the format '....' + Give a decent range of values to allow for flexibility + If a specific value is required, set the default to that value + e.g. if the value should be 5, set the value to '0..5..10' + """), + ) + NickName: Optional[str] = Field( + ..., + description=textwrap.dedent( + """A nickname for the Number Slider, if any, describing what the + 'Number Slider' represents, e.g. 'height', 'x-spacing', 'radius' + etc. + """) ) @@ -124,6 +136,68 @@ class Connection(BaseModel): ) +class StrategyStep(BaseModel): + ComponentName: str + StepDescription: str + + +class Strategy(BaseModel): + """ + Detailed and concise Strategy for creating a grasshopper script. + Make sure to include number sliders for inputs where relevant. + """ + ChainOfThought: List[StrategyStep] = Field( + ..., + description=textwrap.dedent( + """ + step by step plan explaining how the script will + acheive the aim, including the all components used. + Start by defining all inputs, e.g. sliders, points, or panels + Be specific, avoid making vague statements. + This strategy needs to give specific instructions that can easily + be carried out by a novice grasshopper user, without the need to + infer any details + """ + ) + ) + + @field_validator("ChainOfThought") + @classmethod + def validate_components_exist(cls, v): + errors: List[InitErrorDetails] = [] + c: StrategyStep + for c in v: + name_list = [component.Name + for component in valid_components.Components] + if c.ComponentName not in name_list \ + and \ + c.ComponentName not in ['Number Slider', 'Panel', 'Point']: + errors.append(InitErrorDetails( + type=PydanticCustomError( + "", + textwrap.dedent(f""" + The component '{c.ComponentName}' could not be found, + either there is a typo or it does not exist. + Did you mean any of these: {get_k_nearest_components( + k=5, + query=f'${c.ComponentName}: ${c.StepDescription}', + valid_components_with_embeddings=valid_components + )}? + Substitute the component for the actual + component that you require only if they are equivalent! + If no equivalent component is found consider redefining + the strategy and 'ChainOfThought' from scratch! + """) + ) + )) + if len(errors) > 0: + raise ValidationError.from_exception_data( + title='Components', + line_errors=errors, + ) + return v + + class GrasshopperScriptModel(BaseModel): """ A acyclic directed graph representation of a grasshopper script with all @@ -150,6 +224,31 @@ class GrasshopperScriptModel(BaseModel): "grasshopper script" ) + @model_validator(mode='after') + def check_circular_ref(self): + """ + Checks if there are any circular references in the connections + """ + errors: List[InitErrorDetails] = [] + for connection in self.Connections: + from_id = connection.From.Id + to_id = connection.To.Id + if from_id == to_id: + errors.append(InitErrorDetails( + type=PydanticCustomError( + "", + f"Connection from component {from_id} to component " + f"{to_id} is a circular reference. Please remove the " + f"connection." + ) + )) + if len(errors) > 0: + raise ValidationError.from_exception_data( + title=self.__class__.__name__, + line_errors=errors, + ) + return self + @model_validator(mode='after') def validate_parameter_names(self): """ @@ -238,66 +337,6 @@ def get_connection_component_name( return script_component.Name -class Strategy(BaseModel): - """ - Detailed and concise Strategy for creating a grasshopper script. - Make sure to include number sliders for inputs where relevant. - """ - ChainOfThought: str = Field( - ..., - description="step by step rational explaining how the script will " - "acheive the aim, including the all components used." - "Be specific, avoid making vague statements." - "This strategy needs to give specific instructions that can easily" - "be carried out by a novice grasshopper user, without the need to" - "infer any details. Make sure to include number sliders for inputs where relevant." - ) - Components: List[str] = Field( - ..., - description="A list of valid grasshopper components to be added" - "to the configuration" - ) - - @field_validator("Components") - @classmethod - def validate_components_exist(cls, v): - errors: List[InitErrorDetails] = [] - - for c in v: - name_list = [component.Name - for component in valid_components.Components] - if c not in name_list \ - and \ - c not in ['Number Slider', 'Panel', 'Point']: - errors.append(InitErrorDetails( - type=PydanticCustomError( - "", - textwrap.dedent(f""" - The component '{c}' could not be found, - either there is a typo or it does not exist. - Did you mean any of these: {get_k_nearest_components( - k=5, - query=c, - valid_components_with_embeddings=valid_components - )}? - Substitute the component for the actual - component that you require only if they are equivalent! - If no equivalent component is found consider redefining - the strategy and 'ChainOfThought' from scratch! - """) - ) - )) - if len(errors) > 0: - raise ValidationError.from_exception_data( - title='Components', - line_errors=errors, - ) - return v - - - - - def find_valid_component_by_name( valid_components: ValidComponents, name: str, @@ -359,46 +398,33 @@ def find_valid_parameter_by_name( class StrategyRating(BaseModel): - problem_statement_adherance: str = Field( - ..., description=textwrap.dedent( - """ - in words, critically describe how well the script addresses the - expected inputs and outputs, and assumptions, as outlined in the problem - statement. - """ - ) - ) - detail: str = Field( - ..., description=textwrap.dedent( - """ - in words, critically describe whether the strategy provides enough detail - in each step of the plan to achieve the goals - to be easily implemented by a novice user. - """ - ) - ) - validation_errors: str = Field( + """ + A critical and honest evaluation of the strategy. + """ + input_adherence: str = Field( ..., description=textwrap.dedent( """ - In words critically consider very carefully any component validation errors - and whether any substitutions would faithfully represent the - original strategy. Explain each one in turn. + in words, and with reference to 'inputs' in the 'Problem + Description' critically evaluate if all inputs have been + included in the plan. + e.g. as slider, point, or panel components """ ) ) susbstitution_recommendations: Optional[List[str]] = Field( ..., description=textwrap.dedent( """ - The recommended substitutions for the components. - One for each error + For each validation error give one single recommended substitution + for the missing component, choose from the list of valid components """ ) ) - other_advice: Optional[str] = Field( + steps_validity: List[str] = Field( ..., description=textwrap.dedent( """ - any other advice to address any issues mentioned in - reasoning + for each strategy step, in words, critically evaluate whether the + single component can truthefully implement everything in the steps + description. """ ) ) @@ -411,9 +437,14 @@ class StrategyRating(BaseModel): class ProblemStatement(BaseModel): - inputs: List[Union[Panel,NumberSlider,Point]] = Field( - ..., - description="list of all inputs required for the script to function" + inputs: List[Union[Panel, NumberSlider, Point]] = Field( + ..., description=textwrap.dedent( + """ + list of all inputs required for the script to function + e.g. Number Slider, Panel, Point components. + If a value is given in the script description ensure + that it is included in the 'Value' field of the component + """) ) outputs: List[str] = Field( ..., diff --git a/models/test_gh_model.py b/models/test_gh_model.py index 0886352..556701d 100644 --- a/models/test_gh_model.py +++ b/models/test_gh_model.py @@ -89,7 +89,7 @@ def test_gh_model_value_fail(): "Value": "5,3" }, { - "Name": "Slider", + "Name": "Number Slider", "Id": 2, "Value": "5..50..100" } diff --git a/patch_openai/patch_openai.py b/patch_openai/patch_openai.py index 20151f3..57aaa7c 100644 --- a/patch_openai/patch_openai.py +++ b/patch_openai/patch_openai.py @@ -49,7 +49,7 @@ def is_async(func: Callable[..., Any]) -> bool: def call_chat(*args, **kwargs): body = {"kwargs": kwargs, "base_url": str(client.base_url)} - request_url = "https://fastapi-production-e161.up.railway.app/chat" + request_url = "https://fast-api-gules.vercel.app/chat" headers = { "api_key": api_key @@ -59,7 +59,6 @@ def call_chat(*args, **kwargs): url=request_url, json=body, headers=headers, - verify=False ) # Create a ChatCompletion object from the dictionary @@ -85,7 +84,7 @@ def new_create_sync(*args, **kwargs): def call_embeddings(*args, **kwargs): body = {"kwargs": kwargs, "base_url": str(client.base_url)} - request_url = "https://fastapi-production-e161.up.railway.app/embeddings" + request_url = "https://fast-api-gules.vercel.app/embeddings" headers = { "api_key": api_key @@ -95,7 +94,6 @@ def call_embeddings(*args, **kwargs): url=request_url, json=body, headers=headers, - verify=False ) # Create a ChatCompletion object from the dictionary diff --git a/pipeline/pipeline.py b/pipeline/pipeline.py index b33cf75..f71bbae 100644 --- a/pipeline/pipeline.py +++ b/pipeline/pipeline.py @@ -173,7 +173,7 @@ async def pipe_strategy( ) system_prompt: str = get_strategy_system_template(examples) model: str = "gpt-3.5-turbo-1106" - gpt4_turbo: str = "gpt-4-turbo" + gpt4o: str = "gpt-4o" temperature: float = 0 response_model: BaseModel = Strategy messages = [ @@ -187,7 +187,7 @@ async def pipe_strategy( try: # generate initial strategy response: Strategy = await client.chat.completions.create( - model=gpt4_turbo, + model=model, messages=messages, temperature=temperature, response_model=response_model, @@ -217,7 +217,11 @@ async def pipe_strategy( *messages[1:], {"role": "user", "content": textwrap.dedent( """ - Does the above strategy look correct? Evaluate and then score + Does the above strategy look correct? Provide the following, + - input adherance + - substitution recommendations + - validity of each step + - overall score """ )} ] @@ -232,8 +236,8 @@ async def pipe_strategy( # create a new strategy based on feedback if rating_response.score < 5 or error is True: messages.append({ - "role": "user", "content": rating_response.validation_errors + - rating_response.model_dump_json() + "role": "user", + "content": rating_response.model_dump_json() }) response: ChatCompletion = await client.chat.completions.create( model=model, diff --git a/prompts/pipeline_prompts.py b/prompts/pipeline_prompts.py index 956c448..515337c 100644 --- a/prompts/pipeline_prompts.py +++ b/prompts/pipeline_prompts.py @@ -39,12 +39,8 @@ def format_strategy_examples(examples: list[str]) -> str: # Description {description} # Strategy - { - Strategy( - ChainOfThought=script_model.ChainOfThought, - Components=[c.Name for c in script_model.Components] - ).model_dump_json() - } + {script_model.ChainOfThought} + {str([c.Name for c in script_model.Components])} """) + "\n\n" return formatted_examples @@ -93,13 +89,12 @@ def get_grasshopper_script_model_system_template(examples:Examples) -> str: definition. """ -#- You will be provided a problem statement. Include number sliders for the inputs where required. def get_strategy_system_template(examples:Examples) -> str: strategy_system_template = """ You are a Grasshopper3d Expert and are going to help create a Grasshopper - definition. + definition You will be given a description of the script to create, required input and - output, and you may also be given some feedback and advice. + output, and you may also be given some feedback and advice Make sure you follow the expected inputs and outputs. If any advice is provided make sure you consider this carefully in defining @@ -109,35 +104,7 @@ def get_strategy_system_template(examples:Examples) -> str: approach the grasshopper script. - Next provide a list of the essential components required to execute the strategy. - - - - {EXAMPLES} - - """.format(EXAMPLES=format_strategy_examples( - [e.model_dump_json() for e in examples] - )) - - return strategy_system_template - - - -def get_strategy_system_template(examples:Examples) -> str: - strategy_system_template = """ - You are a Grasshopper3d Expert and are going to help create a Grasshopper - definition. - You will be given a description of the script to create, required input and - output, and you may also be given some feedback and advice. - - Make sure you follow the expected inputs and outputs. - If any advice is provided make sure you consider this carefully in defining - your strategy. - - - First, you must provide a concise and well defined strategy for how to - approach the grasshopper script. - - Next provide a list of the essential components required to execute the - strategy. - + {EXAMPLES} @@ -149,7 +116,6 @@ def get_strategy_system_template(examples:Examples) -> str: return strategy_system_template - def get_follow_up_system_template(examples:Examples) -> str: follow_up_system_template = """ You are a Grasshopper3d Expert and are going to help create a Grasshopper @@ -182,10 +148,10 @@ def get_description_strategy_template(user_prompt: str, strategy: Strategy): components: List[ValidComponent] = \ [find_valid_component_by_name( valid_components=valid_components, - name=c, + name=c.ComponentName, errors=[] ) - for c in strategy.Components] + for c in strategy.ChainOfThought] components_str = '' for c in components: components_str += c.model_dump_json() diff --git a/setup.py b/setup.py index df8dd9b..9a5a093 100644 --- a/setup.py +++ b/setup.py @@ -5,4 +5,4 @@ version='0.1', description='The aim is to "flow engineer" a Large Language Model (LLM) powered program to generate Grasshopper3d scripts based on a user prompt as input.', packages=find_packages(), -) \ No newline at end of file +)