From d9ae887c9c4e6255af7c23d20afb36d7d18ec101 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 6 Nov 2024 17:42:20 -0500 Subject: [PATCH 01/30] Add SambaNovaCloud bind tools --- .../chat_models/sambanova.py | 338 +++++++++++------- 1 file changed, 212 insertions(+), 126 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 62bc366486805..20aa641db1353 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -1,10 +1,12 @@ import json -from typing import Any, Dict, Iterator, List, Optional, Tuple +from typing import Any, Callable, Dict, Iterator, List, Literal, Optional, Sequence, Type, Tuple, Union import requests +from langchain_core.tools import BaseTool from langchain_core.callbacks import ( CallbackManagerForLLMRun, ) +from langchain_core.language_models import LanguageModelInput from langchain_core.language_models.chat_models import ( BaseChatModel, generate_from_stream, @@ -20,7 +22,13 @@ ToolMessage, ) from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult +from langchain_core.output_parsers.openai_tools import ( + make_invalid_tool_call, + parse_tool_call, +) +from langchain_core.runnables import Runnable from langchain_core.utils import convert_to_secret_str, get_from_dict_or_env +from langchain_core.utils.function_calling import convert_to_openai_tool from pydantic import Field, SecretStr from requests import Response @@ -142,6 +150,30 @@ class ChatSambaNovaCloud(BaseChatModel): response = chat.ainvoke(messages) await response + + Tool calling: + .. code-block:: python + + from pydantic import BaseModel, Field + + class GetWeather(BaseModel): + '''Get the current weather in a given location''' + + location: str = Field(..., description="The city and state, e.g. Los Angeles, CA") + + llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) + ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") + ai_msg.tool_calls + + .. code-block:: python + + [ + { + 'name': 'GetWeather', + 'args': {'location': 'Los Angeles, CA'}, + 'id': 'call_adf61180ea2b4d228a' + } + ] Token usage: .. code-block:: python @@ -230,135 +262,221 @@ def __init__(self, **kwargs: Any) -> None: ) super().__init__(**kwargs) - def _handle_request( - self, messages_dicts: List[Dict], stop: Optional[List[str]] = None - ) -> Dict[str, Any]: + def bind_tools( + self, + tools: Sequence[Union[Dict[str, Any], Type, Callable, BaseTool]], + *, + tool_choice: Optional[Union[dict, bool, str]] = None, + parallel_tool_calls: Optional[bool] = False, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, BaseMessage]: + """Bind tool-like objects to this chat model + + tool_choice: does not currently support "any", choice like + should be one of ["auto", "none", "required"] + """ + + formatted_tools = [ + convert_to_openai_tool(tool) for tool in tools + ] + + if tool_choice: + if isinstance(tool_choice, str): + # tool_choice is a tool/function name + if tool_choice not in ("auto", "none", "required"): + tool_choice = "auto" + elif isinstance(tool_choice, bool): + if tool_choice: + tool_choice = "required" + elif isinstance(tool_choice, dict): + raise ValueError("tool_choice must be one of ['auto', 'none', 'required']") + else: + raise ValueError( + f"Unrecognized tool_choice type. Expected str, bool" + f"Received: {tool_choice}" + ) + else: + tool_choice = "auto" + kwargs["tool_choice"] = tool_choice + kwargs["parallel_tool_calls"] = parallel_tool_calls + return super().bind(tools=formatted_tools, **kwargs) + + + def _handle_request(self, messages_dicts: List[Dict], stop: Optional[List[str]] = None, streaming: bool = False, **kwargs) -> Response: """ Performs a post request to the LLM API. Args: messages_dicts: List of role / content dicts to use as input. stop: list of stop tokens + streaming: wether to do a streaming call Returns: An iterator of response dicts. """ - data = { - "messages": messages_dicts, - "max_tokens": self.max_tokens, - "stop": stop, - "model": self.model, - "temperature": self.temperature, - "top_p": self.top_p, - "top_k": self.top_k, - } + if streaming: + data = { + 'messages': messages_dicts, + 'max_tokens': self.max_tokens, + 'stop': stop, + 'model': self.model, + 'temperature': self.temperature, + 'top_p': self.top_p, + 'top_k': self.top_k, + 'stream': True, + 'stream_options': self.stream_options, + **kwargs + } + else: + data = { + 'messages': messages_dicts, + 'max_tokens': self.max_tokens, + 'stop': stop, + 'model': self.model, + 'temperature': self.temperature, + 'top_p': self.top_p, + 'top_k': self.top_k, + **kwargs, + } http_session = requests.Session() response = http_session.post( self.sambanova_url, headers={ - "Authorization": f"Bearer {self.sambanova_api_key.get_secret_value()}", - "Content-Type": "application/json", + 'Authorization': f'Bearer {self.sambanova_api_key.get_secret_value()}', + 'Content-Type': 'application/json' }, json=data, + stream = streaming ) if response.status_code != 200: raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}.", - f"{response.text}.", + f'Sambanova /complete call failed with status code ' f'{response.status_code}.', + f'{response.text}.', ) - response_dict = response.json() - if response_dict.get("error"): + return response + + def _process_response(self, response: Response) -> AIMessage: + """ + Process a non streaming response from the api + + Args: + response: A request Response object + + Returns + generation: an AIMessage with model generation + """ + try: + response_dict = response.json() + if response_dict.get('error'): + raise RuntimeError( + f'Sambanova /complete call failed with status code ' f'{response.status_code}.', + f'{response_dict}.', + ) + except Exception as e: raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}.", - f"{response_dict}.", + f"Sambanova /complete call failed couldn't get JSON response {e}" f'response: {response.text}' ) - return response_dict - - def _handle_streaming_request( - self, messages_dicts: List[Dict], stop: Optional[List[str]] = None - ) -> Iterator[Dict]: + content = response_dict['choices'][0]['message'].get('content',"") + if content is None: + content = "" + additional_kwargs: Dict = {} + tool_calls = [] + invalid_tool_calls = [] + raw_tool_calls = response_dict['choices'][0]['message'].get('tool_calls') + if raw_tool_calls: + additional_kwargs["tool_calls"] = raw_tool_calls + for raw_tool_call in raw_tool_calls: + if isinstance(raw_tool_call["function"]["arguments"], dict): + raw_tool_call["function"]["arguments"]=json.dumps(raw_tool_call["function"].get("arguments",{})) + try: + tool_calls.append(parse_tool_call(raw_tool_call, return_id=True)) + except Exception as e: + invalid_tool_calls.append( + make_invalid_tool_call(raw_tool_call, str(e)) + ) + message = AIMessage( + content=content, + additional_kwargs={}, + tool_calls=tool_calls, + invalid_tool_calls=invalid_tool_calls, + response_metadata={ + 'finish_reason': response_dict['choices'][0]['finish_reason'], + 'usage': response_dict.get('usage'), + 'model_name': response_dict['model'], + 'system_fingerprint': response_dict['system_fingerprint'], + 'created': response_dict['created'], + }, + id=response_dict['id'], + ) + return message + + def _process_stream_response(self, response: Response) -> Iterator[BaseMessageChunk]: """ - Performs an streaming post request to the LLM API. + Process a streaming response from the api Args: - messages_dicts: List of role / content dicts to use as input. - stop: list of stop tokens + response: An iterable request Response object Yields: - An iterator of response dicts. + generation: an AIMessageChunk with model partial generation """ try: import sseclient except ImportError: - raise ImportError( - "could not import sseclient library" - "Please install it with `pip install sseclient-py`." - ) - data = { - "messages": messages_dicts, - "max_tokens": self.max_tokens, - "stop": stop, - "model": self.model, - "temperature": self.temperature, - "top_p": self.top_p, - "top_k": self.top_k, - "stream": True, - "stream_options": self.stream_options, - } - http_session = requests.Session() - response = http_session.post( - self.sambanova_url, - headers={ - "Authorization": f"Bearer {self.sambanova_api_key.get_secret_value()}", - "Content-Type": "application/json", - }, - json=data, - stream=True, - ) + raise ImportError('could not import sseclient library' 'Please install it with `pip install sseclient-py`.') client = sseclient.SSEClient(response) - if response.status_code != 200: - raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}." - f"{response.text}." - ) - for event in client.events(): - if event.event == "error_event": + if event.event == 'error_event': raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}." - f"{event.data}." + f'Sambanova /complete call failed with status code ' f'{response.status_code}.' f'{event.data}.' ) try: # check if the response is a final event # in that case event data response is '[DONE]' - if event.data != "[DONE]": + if event.data != '[DONE]': if isinstance(event.data, str): data = json.loads(event.data) else: raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}." - f"{event.data}." + f'Sambanova /complete call failed with status code ' + f'{response.status_code}.' + f'{event.data}.' ) - if data.get("error"): + if data.get('error'): raise RuntimeError( - f"Sambanova /complete call failed with status code " - f"{response.status_code}." - f"{event.data}." + f'Sambanova /complete call failed with status code ' + f'{response.status_code}.' + f'{event.data}.' ) - yield data + if len(data['choices']) > 0: + finish_reason = data['choices'][0].get('finish_reason') + content = data['choices'][0]['delta']['content'] + id = data['id'] + chunk = AIMessageChunk(content=content, id=id, additional_kwargs={}) + else: + content = '' + id = data['id'] + metadata = { + 'finish_reason': finish_reason, + 'usage': data.get('usage'), + 'model_name': data['model'], + 'system_fingerprint': data['system_fingerprint'], + 'created': data['created'], + } + chunk = AIMessageChunk( + content=content, + id=id, + response_metadata=metadata, + additional_kwargs={}, + ) + yield chunk + except Exception as e: - raise RuntimeError( - f"Error getting content chunk raw streamed response: {e}" - f"data: {event.data}" - ) + raise RuntimeError(f'Error getting content chunk raw streamed response: {e}' f'data: {event.data}') + def _generate( self, @@ -384,27 +502,18 @@ def _generate( result: ChatResult with model generation """ if self.streaming: - stream_iter = self._stream( - messages, stop=stop, run_manager=run_manager, **kwargs - ) + stream_iter = self._stream(messages, stop=stop, run_manager=run_manager, **kwargs) if stream_iter: return generate_from_stream(stream_iter) messages_dicts = _create_message_dicts(messages) - response = self._handle_request(messages_dicts, stop) - message = AIMessage( - content=response["choices"][0]["message"]["content"], - additional_kwargs={}, - response_metadata={ - "finish_reason": response["choices"][0]["finish_reason"], - "usage": response.get("usage"), - "model_name": response["model"], - "system_fingerprint": response["system_fingerprint"], - "created": response["created"], - }, - id=response["id"], + response = self._handle_request(messages_dicts, stop, streaming=False, **kwargs) + message = self._process_response(response) + generation = ChatGeneration( + message=message, + generation_info={ + "finish_reason": message.response_metadata["finish_reason"] + } ) - - generation = ChatGeneration(message=message) return ChatResult(generations=[generation]) def _stream( @@ -431,34 +540,11 @@ def _stream( chunk: ChatGenerationChunk with model partial generation """ messages_dicts = _create_message_dicts(messages) - finish_reason = None - for partial_response in self._handle_streaming_request(messages_dicts, stop): - if len(partial_response["choices"]) > 0: - finish_reason = partial_response["choices"][0].get("finish_reason") - content = partial_response["choices"][0]["delta"]["content"] - id = partial_response["id"] - chunk = ChatGenerationChunk( - message=AIMessageChunk(content=content, id=id, additional_kwargs={}) - ) - else: - content = "" - id = partial_response["id"] - metadata = { - "finish_reason": finish_reason, - "usage": partial_response.get("usage"), - "model_name": partial_response["model"], - "system_fingerprint": partial_response["system_fingerprint"], - "created": partial_response["created"], - } - chunk = ChatGenerationChunk( - message=AIMessageChunk( - content=content, - id=id, - response_metadata=metadata, - additional_kwargs={}, - ) - ) - + response = self._handle_request(messages_dicts, stop, streaming = True, **kwargs) + for ai_message_chunk in self._process_stream_response(response): + chunk = ChatGenerationChunk( + message=ai_message_chunk + ) if run_manager: run_manager.on_llm_new_token(chunk.text, chunk=chunk) yield chunk From 4e731940a3a30f69c79d7016d9f852250bdc64d2 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 7 Nov 2024 09:37:31 -0500 Subject: [PATCH 02/30] fmt --- .../chat_models/sambanova.py | 209 ++++++++++-------- 1 file changed, 122 insertions(+), 87 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 20aa641db1353..a7fafa93ed1aa 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -1,8 +1,18 @@ import json -from typing import Any, Callable, Dict, Iterator, List, Literal, Optional, Sequence, Type, Tuple, Union +from typing import ( + Any, + Callable, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, + Union, +) import requests -from langchain_core.tools import BaseTool from langchain_core.callbacks import ( CallbackManagerForLLMRun, ) @@ -21,12 +31,13 @@ SystemMessage, ToolMessage, ) -from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult from langchain_core.output_parsers.openai_tools import ( make_invalid_tool_call, parse_tool_call, ) +from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult from langchain_core.runnables import Runnable +from langchain_core.tools import BaseTool from langchain_core.utils import convert_to_secret_str, get_from_dict_or_env from langchain_core.utils.function_calling import convert_to_openai_tool from pydantic import Field, SecretStr @@ -150,7 +161,7 @@ class ChatSambaNovaCloud(BaseChatModel): response = chat.ainvoke(messages) await response - + Tool calling: .. code-block:: python @@ -159,7 +170,10 @@ class ChatSambaNovaCloud(BaseChatModel): class GetWeather(BaseModel): '''Get the current weather in a given location''' - location: str = Field(..., description="The city and state, e.g. Los Angeles, CA") + location: str = Field( + ..., + description="The city and state, e.g. Los Angeles, CA" + ) llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") @@ -269,17 +283,15 @@ def bind_tools( tool_choice: Optional[Union[dict, bool, str]] = None, parallel_tool_calls: Optional[bool] = False, **kwargs: Any, - ) -> Runnable[LanguageModelInput, BaseMessage]: + ) -> Runnable[LanguageModelInput, BaseMessage]: """Bind tool-like objects to this chat model - tool_choice: does not currently support "any", choice like - should be one of ["auto", "none", "required"] + tool_choice: does not currently support "any", choice like + should be one of ["auto", "none", "required"] """ - - formatted_tools = [ - convert_to_openai_tool(tool) for tool in tools - ] - + + formatted_tools = [convert_to_openai_tool(tool) for tool in tools] + if tool_choice: if isinstance(tool_choice, str): # tool_choice is a tool/function name @@ -289,7 +301,9 @@ def bind_tools( if tool_choice: tool_choice = "required" elif isinstance(tool_choice, dict): - raise ValueError("tool_choice must be one of ['auto', 'none', 'required']") + raise ValueError( + "tool_choice must be one of ['auto', 'none', 'required']" + ) else: raise ValueError( f"Unrecognized tool_choice type. Expected str, bool" @@ -301,8 +315,13 @@ def bind_tools( kwargs["parallel_tool_calls"] = parallel_tool_calls return super().bind(tools=formatted_tools, **kwargs) - - def _handle_request(self, messages_dicts: List[Dict], stop: Optional[List[str]] = None, streaming: bool = False, **kwargs) -> Response: + def _handle_request( + self, + messages_dicts: List[Dict[str, Any]], + stop: Optional[List[str]] = None, + streaming: bool = False, + **kwargs: Any, + ) -> Response: """ Performs a post request to the LLM API. @@ -316,45 +335,46 @@ def _handle_request(self, messages_dicts: List[Dict], stop: Optional[List[str]] """ if streaming: data = { - 'messages': messages_dicts, - 'max_tokens': self.max_tokens, - 'stop': stop, - 'model': self.model, - 'temperature': self.temperature, - 'top_p': self.top_p, - 'top_k': self.top_k, - 'stream': True, - 'stream_options': self.stream_options, - **kwargs + "messages": messages_dicts, + "max_tokens": self.max_tokens, + "stop": stop, + "model": self.model, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "stream": True, + "stream_options": self.stream_options, + **kwargs, } else: data = { - 'messages': messages_dicts, - 'max_tokens': self.max_tokens, - 'stop': stop, - 'model': self.model, - 'temperature': self.temperature, - 'top_p': self.top_p, - 'top_k': self.top_k, + "messages": messages_dicts, + "max_tokens": self.max_tokens, + "stop": stop, + "model": self.model, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, **kwargs, } http_session = requests.Session() response = http_session.post( self.sambanova_url, headers={ - 'Authorization': f'Bearer {self.sambanova_api_key.get_secret_value()}', - 'Content-Type': 'application/json' + "Authorization": f"Bearer {self.sambanova_api_key.get_secret_value()}", + "Content-Type": "application/json", }, json=data, - stream = streaming + stream=streaming, ) if response.status_code != 200: raise RuntimeError( - f'Sambanova /complete call failed with status code ' f'{response.status_code}.', - f'{response.text}.', + f"Sambanova /complete call failed with status code " + f"{response.status_code}.", + f"{response.text}.", ) return response - + def _process_response(self, response: Response) -> AIMessage: """ Process a non streaming response from the api @@ -367,27 +387,31 @@ def _process_response(self, response: Response) -> AIMessage: """ try: response_dict = response.json() - if response_dict.get('error'): + if response_dict.get("error"): raise RuntimeError( - f'Sambanova /complete call failed with status code ' f'{response.status_code}.', - f'{response_dict}.', + f"Sambanova /complete call failed with status code " + f"{response.status_code}.", + f"{response_dict}.", ) except Exception as e: raise RuntimeError( - f"Sambanova /complete call failed couldn't get JSON response {e}" f'response: {response.text}' + f"Sambanova /complete call failed couldn't get JSON response {e}" + f"response: {response.text}" ) - content = response_dict['choices'][0]['message'].get('content',"") + content = response_dict["choices"][0]["message"].get("content", "") if content is None: content = "" additional_kwargs: Dict = {} tool_calls = [] invalid_tool_calls = [] - raw_tool_calls = response_dict['choices'][0]['message'].get('tool_calls') + raw_tool_calls = response_dict["choices"][0]["message"].get("tool_calls") if raw_tool_calls: additional_kwargs["tool_calls"] = raw_tool_calls for raw_tool_call in raw_tool_calls: if isinstance(raw_tool_call["function"]["arguments"], dict): - raw_tool_call["function"]["arguments"]=json.dumps(raw_tool_call["function"].get("arguments",{})) + raw_tool_call["function"]["arguments"] = json.dumps( + raw_tool_call["function"].get("arguments", {}) + ) try: tool_calls.append(parse_tool_call(raw_tool_call, return_id=True)) except Exception as e: @@ -400,17 +424,19 @@ def _process_response(self, response: Response) -> AIMessage: tool_calls=tool_calls, invalid_tool_calls=invalid_tool_calls, response_metadata={ - 'finish_reason': response_dict['choices'][0]['finish_reason'], - 'usage': response_dict.get('usage'), - 'model_name': response_dict['model'], - 'system_fingerprint': response_dict['system_fingerprint'], - 'created': response_dict['created'], + "finish_reason": response_dict["choices"][0]["finish_reason"], + "usage": response_dict.get("usage"), + "model_name": response_dict["model"], + "system_fingerprint": response_dict["system_fingerprint"], + "created": response_dict["created"], }, - id=response_dict['id'], + id=response_dict["id"], ) return message - - def _process_stream_response(self, response: Response) -> Iterator[BaseMessageChunk]: + + def _process_stream_response( + self, response: Response + ) -> Iterator[BaseMessageChunk]: """ Process a streaming response from the api @@ -423,48 +449,55 @@ def _process_stream_response(self, response: Response) -> Iterator[BaseMessageCh try: import sseclient except ImportError: - raise ImportError('could not import sseclient library' 'Please install it with `pip install sseclient-py`.') + raise ImportError( + "could not import sseclient library" + "Please install it with `pip install sseclient-py`." + ) client = sseclient.SSEClient(response) for event in client.events(): - if event.event == 'error_event': + if event.event == "error_event": raise RuntimeError( - f'Sambanova /complete call failed with status code ' f'{response.status_code}.' f'{event.data}.' + f"Sambanova /complete call failed with status code " + f"{response.status_code}." + f"{event.data}." ) try: # check if the response is a final event # in that case event data response is '[DONE]' - if event.data != '[DONE]': + if event.data != "[DONE]": if isinstance(event.data, str): data = json.loads(event.data) else: raise RuntimeError( - f'Sambanova /complete call failed with status code ' - f'{response.status_code}.' - f'{event.data}.' + f"Sambanova /complete call failed with status code " + f"{response.status_code}." + f"{event.data}." ) - if data.get('error'): + if data.get("error"): raise RuntimeError( - f'Sambanova /complete call failed with status code ' - f'{response.status_code}.' - f'{event.data}.' + f"Sambanova /complete call failed with status code " + f"{response.status_code}." + f"{event.data}." + ) + if len(data["choices"]) > 0: + finish_reason = data["choices"][0].get("finish_reason") + content = data["choices"][0]["delta"]["content"] + id = data["id"] + chunk = AIMessageChunk( + content=content, id=id, additional_kwargs={} ) - if len(data['choices']) > 0: - finish_reason = data['choices'][0].get('finish_reason') - content = data['choices'][0]['delta']['content'] - id = data['id'] - chunk = AIMessageChunk(content=content, id=id, additional_kwargs={}) else: - content = '' - id = data['id'] + content = "" + id = data["id"] metadata = { - 'finish_reason': finish_reason, - 'usage': data.get('usage'), - 'model_name': data['model'], - 'system_fingerprint': data['system_fingerprint'], - 'created': data['created'], + "finish_reason": finish_reason, + "usage": data.get("usage"), + "model_name": data["model"], + "system_fingerprint": data["system_fingerprint"], + "created": data["created"], } chunk = AIMessageChunk( content=content, @@ -473,10 +506,12 @@ def _process_stream_response(self, response: Response) -> Iterator[BaseMessageCh additional_kwargs={}, ) yield chunk - - except Exception as e: - raise RuntimeError(f'Error getting content chunk raw streamed response: {e}' f'data: {event.data}') + except Exception as e: + raise RuntimeError( + f"Error getting content chunk raw streamed response: {e}" + f"data: {event.data}" + ) def _generate( self, @@ -502,17 +537,19 @@ def _generate( result: ChatResult with model generation """ if self.streaming: - stream_iter = self._stream(messages, stop=stop, run_manager=run_manager, **kwargs) + stream_iter = self._stream( + messages, stop=stop, run_manager=run_manager, **kwargs + ) if stream_iter: return generate_from_stream(stream_iter) messages_dicts = _create_message_dicts(messages) response = self._handle_request(messages_dicts, stop, streaming=False, **kwargs) message = self._process_response(response) generation = ChatGeneration( - message=message, + message=message, generation_info={ "finish_reason": message.response_metadata["finish_reason"] - } + }, ) return ChatResult(generations=[generation]) @@ -540,11 +577,9 @@ def _stream( chunk: ChatGenerationChunk with model partial generation """ messages_dicts = _create_message_dicts(messages) - response = self._handle_request(messages_dicts, stop, streaming = True, **kwargs) + response = self._handle_request(messages_dicts, stop, streaming=True, **kwargs) for ai_message_chunk in self._process_stream_response(response): - chunk = ChatGenerationChunk( - message=ai_message_chunk - ) + chunk = ChatGenerationChunk(message=ai_message_chunk) if run_manager: run_manager.on_llm_new_token(chunk.text, chunk=chunk) yield chunk From f2d371968e0de99533d1ba01886ef0f068228dc1 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 7 Nov 2024 10:50:02 -0500 Subject: [PATCH 03/30] minor changes typing --- .../langchain_community/chat_models/sambanova.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index a7fafa93ed1aa..2310aba5800a4 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -226,7 +226,7 @@ class GetWeather(BaseModel): top_k: Optional[int] = Field(default=None) """model top k""" - stream_options: dict = Field(default={"include_usage": True}) + stream_options: Dict[str, Any] = Field(default={"include_usage": True}) """stream options, include usage to get generation metrics""" class Config: @@ -278,9 +278,9 @@ def __init__(self, **kwargs: Any) -> None: def bind_tools( self, - tools: Sequence[Union[Dict[str, Any], Type, Callable, BaseTool]], + tools: Sequence[Union[Dict[str, Any], Type[Any], Callable[..., Any], BaseTool]], *, - tool_choice: Optional[Union[dict, bool, str]] = None, + tool_choice: Optional[Union[Dict[str, Any], bool, str]] = None, parallel_tool_calls: Optional[bool] = False, **kwargs: Any, ) -> Runnable[LanguageModelInput, BaseMessage]: @@ -401,7 +401,7 @@ def _process_response(self, response: Response) -> AIMessage: content = response_dict["choices"][0]["message"].get("content", "") if content is None: content = "" - additional_kwargs: Dict = {} + additional_kwargs: Dict[str, Any] = {} tool_calls = [] invalid_tool_calls = [] raw_tool_calls = response_dict["choices"][0]["message"].get("tool_calls") @@ -738,10 +738,10 @@ class ChatSambaStudio(BaseChatModel): process_prompt: Optional[bool] = Field(default=True) """whether process prompt (for CoE generic v1 and v2 endpoints)""" - stream_options: dict = Field(default={"include_usage": True}) + stream_options: Dict[str, Any] = Field(default={"include_usage": True}) """stream options, include usage to get generation metrics""" - special_tokens: dict = Field( + special_tokens: Dict[str, Any] = Field( default={ "start": "<|begin_of_text|>", "start_role": "<|begin_of_text|><|start_header_id|>{role}<|end_header_id|>", From 93543b8c9802f5b2b4cd5f0e837dd6eca80a40c8 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 7 Nov 2024 12:46:42 -0500 Subject: [PATCH 04/30] add tool calling in docs --- docs/docs/integrations/chat/sambanova.ipynb | 121 +++++++++++++++++--- 1 file changed, 106 insertions(+), 15 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index 7f00e09467979..23c1c243fe9f6 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -34,7 +34,7 @@ "\n", "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", + "| ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", "\n", "## Setup\n", "\n", @@ -116,14 +116,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", "\n", "llm = ChatSambaNovaCloud(\n", - " model=\"llama3-405b\", max_tokens=1024, temperature=0.7, top_k=1, top_p=0.01\n", + " model=\"Meta-Llama-3.1-70B-Instruct\", max_tokens=1024, temperature=0.7, top_k=1, top_p=0.01\n", ")" ] }, @@ -142,7 +142,7 @@ { "data": { "text/plain": [ - "AIMessage(content=\"J'adore la programmation.\", response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 11, 'completion_tokens': 9, 'completion_tokens_after_first_per_sec': 97.07042823956884, 'completion_tokens_after_first_per_sec_first_ten': 276.3343994441849, 'completion_tokens_per_sec': 23.775192800224037, 'end_time': 1726158364.7954874, 'is_last_response': True, 'prompt_tokens': 56, 'start_time': 1726158364.3670964, 'time_to_first_token': 0.3459765911102295, 'total_latency': 0.3785458261316473, 'total_tokens': 65, 'total_tokens_per_sec': 171.70972577939582}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726158364}, id='7154b676-9d5a-4b1a-a425-73bbe69f28fc')" + "AIMessage(content=\"J'adore la programmation.\", additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 14, 'completion_tokens': 8, 'completion_tokens_after_first_per_sec': 164.48250980392157, 'completion_tokens_after_first_per_sec_first_ten': 854.0507017671442, 'completion_tokens_per_sec': 33.17909116130268, 'end_time': 1731001001.274775, 'is_last_response': True, 'prompt_tokens': 55, 'start_time': 1731001001.0004687, 'time_to_first_token': 0.2317485809326172, 'total_latency': 0.24111570630754744, 'total_tokens': 63, 'total_tokens_per_sec': 261.2853428952586}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001000}, id='415968fc-1a58-47e0-a8a3-7cb80383980b')" ] }, "execution_count": 3, @@ -196,7 +196,7 @@ { "data": { "text/plain": [ - "AIMessage(content='Ich liebe Programmieren.', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 11, 'completion_tokens': 6, 'completion_tokens_after_first_per_sec': 47.80258530102961, 'completion_tokens_after_first_per_sec_first_ten': 215.59002827036753, 'completion_tokens_per_sec': 5.263977583489829, 'end_time': 1726158506.3777263, 'is_last_response': True, 'prompt_tokens': 51, 'start_time': 1726158505.1611376, 'time_to_first_token': 1.1119918823242188, 'total_latency': 1.1398224830627441, 'total_tokens': 57, 'total_tokens_per_sec': 50.00778704315337}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726158505}, id='226471ac-8c52-44bb-baa7-f9d2f8c54477')" + "AIMessage(content='Ich liebe das Programmieren.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 2.3333333333333335, 'completion_tokens': 6, 'completion_tokens_after_first_per_sec': 100.67263842084546, 'completion_tokens_after_first_per_sec_first_ten': 205.99550965424336, 'completion_tokens_per_sec': 36.087854367623336, 'end_time': 1731001005.4113064, 'is_last_response': True, 'prompt_tokens': 50, 'start_time': 1731001005.2245064, 'time_to_first_token': 0.13713407516479492, 'total_latency': 0.1662609236580985, 'total_tokens': 56, 'total_tokens_per_sec': 336.8199740978178}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001005}, id='0990bec1-1721-41f9-a14c-d0bea8ac6263')" ] }, "execution_count": 5, @@ -243,17 +243,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "Yer lookin' fer some info on owls, eh? Alright then, matey, settle yerself down with a pint o' grog and listen close.\n", - "\n", - "Owls be nocturnal birds o' prey, meanin' they do most o' their huntin' at night. They got big, round eyes that be perfect fer seein' in the dark, like a trusty lantern on a dark sea. Their ears be sharp as a cutlass, too, helpin' 'em pinpoint the slightest sound o' a scurvy rodent scurryin' through the underbrush.\n", + "Yer lookin' fer some knowledge about owls, eh? Alright then, matey, settle yerself down with a pint o' grog and listen close.\n", "\n", - "These birds be known fer their silent flight, like a ghost ship sailin' through the night. Their feathers be special, with a soft, fringed edge that helps 'em sneak up on their prey. And when they strike, it be swift and deadly, like a pirate's sword.\n", + "Owls be a fascinatin' lot, with their big round eyes and silent wings. They be birds o' prey, which means they hunt other creatures fer food. There be over 220 species o' owls, rangin' in size from the tiny Elf Owl (which be smaller than a parrot) to the Great Grey Owl (which be as big as a small eagle).\n", "\n", - "Owls be found all over the world, from the frozen tundras o' the north to the scorching deserts o' the south. They come in all shapes and sizes, from the tiny elf owl to the great grey owl, which be as big as a small dog.\n", + "Here be some o' the most interestin' facts about owls:\n", "\n", - "Now, I know what ye be thinkin', \"Pirate, what about their hootin'?\" Aye, owls be famous fer their hoots, which be a form o' communication. They use different hoots to warn off predators, attract a mate, or even just to say, \"Shiver me timbers, I be happy to be alive!\"\n", + "1. **Night vision**: Owls have eyes that be specially designed fer seein' in the dark. Their eyes be big and round, with a reflective layer called the tapetum lucidum that helps 'em see in low light.\n", + "2. **Silent flyers**: Owls have special feathers on their wings that help 'em fly silently. This be useful fer sneakin' up on prey, matey!\n", + "3. **Turnin' heads**: Owls can turn their heads up to 270 degrees, which be useful fer seein' what be goin' on around 'em without havin' to move their whole body.\n", + "4. **Huntin' skills**: Owls be skilled hunters, with sharp talons and strong wings. They can catch prey in mid-air, and some species can even catch fish right out o' the water!\n", + "5. **Monogamous**: Many owl species be monogamous, which means they mate fer life. Some species even stay together fer many years, which be a rare thing in the animal kingdom.\n", "\n", - "So there ye have it, me hearty. Owls be fascinatin' creatures, and I hope ye found this info as interestin' as a chest overflowin' with gold doubloons. Fair winds and following seas!" + "So there ye have it, matey! Owls be amazin' creatures, and there be much more to learn about 'em. Now go forth and spread the word about these fascinatin' birds!" ] } ], @@ -283,7 +285,7 @@ { "data": { "text/plain": [ - "AIMessage(content='The capital of France is Paris.', response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 13, 'completion_tokens': 8, 'completion_tokens_after_first_per_sec': 86.00726488715989, 'completion_tokens_after_first_per_sec_first_ten': 326.92555640828857, 'completion_tokens_per_sec': 21.74539360394493, 'end_time': 1726159287.9987085, 'is_last_response': True, 'prompt_tokens': 43, 'start_time': 1726159287.5738964, 'time_to_first_token': 0.34342360496520996, 'total_latency': 0.36789400760944074, 'total_tokens': 51, 'total_tokens_per_sec': 138.62688422514893}, 'model_name': 'Meta-Llama-3.1-405B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1726159287}, id='9b4ef015-50a2-434b-b980-29f8aa90c3e8')" + "AIMessage(content='The capital of France is Paris.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 1, 'completion_tokens': 7, 'completion_tokens_after_first_per_sec': 326.8246386410566, 'completion_tokens_after_first_per_sec_first_ten': 0, 'completion_tokens_per_sec': 28.56260299906024, 'end_time': 1731001010.9238367, 'is_last_response': True, 'prompt_tokens': 42, 'start_time': 1731001010.678761, 'time_to_first_token': 0.22671723365783691, 'total_latency': 0.24507570266723633, 'total_tokens': 49, 'total_tokens_per_sec': 199.93822099342168}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001010}, id='c17f7e9b-2db4-4c53-bf1a-980c773cf554')" ] }, "execution_count": 7, @@ -321,7 +323,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Quantum computers use quantum bits (qubits) to process vast amounts of data simultaneously, leveraging quantum mechanics to solve complex problems exponentially faster than classical computers." + "Quantum computers use quantum bits (qubits) to process info, leveraging superposition and entanglement to perform calculations exponentially faster than classical computers for certain complex problems." ] } ], @@ -340,6 +342,95 @@ " print(chunk.content, end=\"\", flush=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool calling\n", + "\n", + "SambaNovaCloud supports tool calling" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from langchain_core.messages import (\n", + " HumanMessage,\n", + " ToolMessage,\n", + " SystemMessage\n", + ")\n", + "from langchain_core.tools import tool\n", + "\n", + "@tool\n", + "def get_time(kind: str = 'both') -> str:\n", + " \"\"\"Returns current date, current time or both.\n", + " Args:\n", + " kind: date, time or both\n", + " \"\"\"\n", + " if kind == 'date':\n", + " date = datetime.now().strftime('%m/%d/%Y')\n", + " return f'Current date: {date}'\n", + " elif kind == 'time':\n", + " time = datetime.now().strftime('%H:%M:%S')\n", + " return f'Current time: {time}'\n", + " else:\n", + " date = datetime.now().strftime('%m/%d/%Y')\n", + " time = datetime.now().strftime('%H:%M:%S')\n", + " return f'Current date: {date}, Current time: {time}'\n", + " \n", + "def invoke_tools(tool_calls, messages):\n", + " for tool_call in tool_calls:\n", + " selected_tool = {\"get_time\": get_time}[tool_call[\"name\"].lower()]\n", + " tool_output = selected_tool.invoke(tool_call[\"args\"])\n", + " print(f\"Tool output: {tool_output}\")\n", + " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", + " return messages\n", + "\n", + "tools = [get_time]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm_with_tools = llm.bind_tools(tools=tools)\n", + "messages = [\n", + " HumanMessage(content=\"I need to schedule a meeting for two weeks from today. Can you tell me the exact date of the meeting?\")\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Intermediate model response: [{'name': 'get_time', 'args': {'kind': 'date'}, 'id': 'call_4092d5dd21cd4eb494', 'type': 'tool_call'}]\n", + "Tool output: Current date: 11/07/2024\n", + "final response: The meeting will be exactly two weeks from today, which would be 25/07/2024.\n" + ] + } + ], + "source": [ + "response = llm_with_tools.invoke(messages)\n", + "if response.tool_calls:\n", + " print(f\"Intermediate model response: {response.tool_calls}\")\n", + " messages.append(response)\n", + " messages = invoke_tools(response.tool_calls, messages)\n", + "response = llm.invoke(messages)\n", + "\n", + "print(f\"final response: {response.content}\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -366,7 +457,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.9.20" } }, "nbformat": 4, From 6050c7cd07f4d70bfe4f90523777aaf91d9e809f Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 7 Nov 2024 12:50:22 -0500 Subject: [PATCH 05/30] fmt --- docs/docs/integrations/chat/sambanova.ipynb | 44 ++++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index 23c1c243fe9f6..c41d37cfa67df 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -123,7 +123,11 @@ "from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", "\n", "llm = ChatSambaNovaCloud(\n", - " model=\"Meta-Llama-3.1-70B-Instruct\", max_tokens=1024, temperature=0.7, top_k=1, top_p=0.01\n", + " model=\"Meta-Llama-3.1-70B-Instruct\",\n", + " max_tokens=1024,\n", + " temperature=0.7,\n", + " top_k=1,\n", + " top_p=0.01,\n", ")" ] }, @@ -358,30 +362,29 @@ "outputs": [], "source": [ "from datetime import datetime\n", - "from langchain_core.messages import (\n", - " HumanMessage,\n", - " ToolMessage,\n", - " SystemMessage\n", - ")\n", + "\n", + "from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage\n", "from langchain_core.tools import tool\n", "\n", + "\n", "@tool\n", - "def get_time(kind: str = 'both') -> str:\n", + "def get_time(kind: str = \"both\") -> str:\n", " \"\"\"Returns current date, current time or both.\n", " Args:\n", " kind: date, time or both\n", " \"\"\"\n", - " if kind == 'date':\n", - " date = datetime.now().strftime('%m/%d/%Y')\n", - " return f'Current date: {date}'\n", - " elif kind == 'time':\n", - " time = datetime.now().strftime('%H:%M:%S')\n", - " return f'Current time: {time}'\n", + " if kind == \"date\":\n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " return f\"Current date: {date}\"\n", + " elif kind == \"time\":\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current time: {time}\"\n", " else:\n", - " date = datetime.now().strftime('%m/%d/%Y')\n", - " time = datetime.now().strftime('%H:%M:%S')\n", - " return f'Current date: {date}, Current time: {time}'\n", - " \n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current date: {date}, Current time: {time}\"\n", + "\n", + "\n", "def invoke_tools(tool_calls, messages):\n", " for tool_call in tool_calls:\n", " selected_tool = {\"get_time\": get_time}[tool_call[\"name\"].lower()]\n", @@ -390,6 +393,7 @@ " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", " return messages\n", "\n", + "\n", "tools = [get_time]" ] }, @@ -401,8 +405,10 @@ "source": [ "llm_with_tools = llm.bind_tools(tools=tools)\n", "messages = [\n", - " HumanMessage(content=\"I need to schedule a meeting for two weeks from today. Can you tell me the exact date of the meeting?\")\n", - " ]" + " HumanMessage(\n", + " content=\"I need to schedule a meeting for two weeks from today. Can you tell me the exact date of the meeting?\"\n", + " )\n", + "]" ] }, { From 8ec7ab3012143d022d684b6e178f1ac748636e89 Mon Sep 17 00:00:00 2001 From: Jorge Piedrahita Ortiz Date: Thu, 7 Nov 2024 18:13:34 -0500 Subject: [PATCH 06/30] minor change --- libs/community/langchain_community/chat_models/sambanova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 2310aba5800a4..ff5ddd3f2e5ad 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -420,7 +420,7 @@ def _process_response(self, response: Response) -> AIMessage: ) message = AIMessage( content=content, - additional_kwargs={}, + additional_kwargs=additional_kwargs, tool_calls=tool_calls, invalid_tool_calls=invalid_tool_calls, response_metadata={ From e419b01a39fa1578320b2eac56b0647cb6d73c75 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Fri, 8 Nov 2024 16:14:35 -0500 Subject: [PATCH 07/30] add bind tools sambastudio chat model --- .../chat_models/sambanova.py | 123 +++++++++++++++++- 1 file changed, 119 insertions(+), 4 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index ff5ddd3f2e5ad..6df9c07844fb6 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -689,6 +689,33 @@ class ChatSambaStudio(BaseChatModel): response = chat.ainvoke(messages) await response + Tool calling: + .. code-block:: python + + from pydantic import BaseModel, Field + + class GetWeather(BaseModel): + '''Get the current weather in a given location''' + + location: str = Field( + ..., + description="The city and state, e.g. Los Angeles, CA" + ) + + llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) + ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") + ai_msg.tool_calls + + .. code-block:: python + + [ + { + 'name': 'GetWeather', + 'args': {'location': 'Los Angeles, CA'}, + 'id': 'call_adf61180ea2b4d228a' + } + ] + Token usage: .. code-block:: python response = chat.invoke(messages) @@ -812,6 +839,45 @@ def __init__(self, **kwargs: Any) -> None: ) super().__init__(**kwargs) + def bind_tools( + self, + tools: Sequence[Union[Dict[str, Any], Type[Any], Callable[..., Any], BaseTool]], + *, + tool_choice: Optional[Union[Dict[str, Any], bool, str]] = None, + parallel_tool_calls: Optional[bool] = False, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, BaseMessage]: + """Bind tool-like objects to this chat model + + tool_choice: does not currently support "any", choice like + should be one of ["auto", "none", "required"] + """ + + formatted_tools = [convert_to_openai_tool(tool) for tool in tools] + + if tool_choice: + if isinstance(tool_choice, str): + # tool_choice is a tool/function name + if tool_choice not in ("auto", "none", "required"): + tool_choice = "auto" + elif isinstance(tool_choice, bool): + if tool_choice: + tool_choice = "required" + elif isinstance(tool_choice, dict): + raise ValueError( + "tool_choice must be one of ['auto', 'none', 'required']" + ) + else: + raise ValueError( + f"Unrecognized tool_choice type. Expected str, bool" + f"Received: {tool_choice}" + ) + else: + tool_choice = "auto" + kwargs["tool_choice"] = tool_choice + kwargs["parallel_tool_calls"] = parallel_tool_calls + return super().bind(tools=formatted_tools, **kwargs) + def _get_role(self, message: BaseMessage) -> str: """ Get the role of LangChain BaseMessage @@ -907,6 +973,7 @@ def _handle_request( messages: List[BaseMessage], stop: Optional[List[str]] = None, streaming: Optional[bool] = False, + **kwargs: Any, ) -> Response: """ Performs a post request to the LLM API. @@ -933,6 +1000,7 @@ def _handle_request( "top_k": self.top_k, "stream": streaming, "stream_options": self.stream_options, + **kwargs, } data = {key: value for key, value in data.items() if value is not None} headers = { @@ -952,6 +1020,7 @@ def _handle_request( "top_p": self.top_p, "top_k": self.top_k, "do_sample": self.do_sample, + **kwargs, } if self.model_kwargs is not None: params = {**params, **self.model_kwargs} @@ -961,6 +1030,11 @@ def _handle_request( # create request payload for generic v1 API elif "api/predict/generic" in self.sambastudio_url: + if "tools" in kwargs.keys(): + raise NotImplementedError( + "tool calling not supported in API Generic V1, " + "switch to OpenAI compatible API or Generic V2 API" + ) params = { "select_expert": self.model, "process_prompt": self.process_prompt, @@ -969,6 +1043,7 @@ def _handle_request( "top_p": self.top_p, "top_k": self.top_k, "do_sample": self.do_sample, + **kwargs, } if self.model_kwargs is not None: params = {**params, **self.model_kwargs} @@ -1032,9 +1107,15 @@ def _process_response(self, response: Response) -> AIMessage: f"response: {response.text}" ) + additional_kwargs: Dict[str, Any] = {} + tool_calls = [] + invalid_tool_calls = [] + # process response payload for openai compatible API if "openai" in self.sambastudio_url: - content = response_dict["choices"][0]["message"]["content"] + content = response_dict["choices"][0]["message"].get("content", "") + if content is None: + content = "" id = response_dict["id"] response_metadata = { "finish_reason": response_dict["choices"][0]["finish_reason"], @@ -1043,12 +1124,44 @@ def _process_response(self, response: Response) -> AIMessage: "system_fingerprint": response_dict["system_fingerprint"], "created": response_dict["created"], } + raw_tool_calls = response_dict["choices"][0]["message"].get("tool_calls") + if raw_tool_calls: + additional_kwargs["tool_calls"] = raw_tool_calls + for raw_tool_call in raw_tool_calls: + if isinstance(raw_tool_call["function"]["arguments"], dict): + raw_tool_call["function"]["arguments"] = json.dumps( + raw_tool_call["function"].get("arguments", {}) + ) + try: + tool_calls.append( + parse_tool_call(raw_tool_call, return_id=True) + ) + except Exception as e: + invalid_tool_calls.append( + make_invalid_tool_call(raw_tool_call, str(e)) + ) # process response payload for generic v2 API elif "api/v2/predict/generic" in self.sambastudio_url: content = response_dict["items"][0]["value"]["completion"] id = response_dict["items"][0]["id"] response_metadata = response_dict["items"][0] + raw_tool_calls = response_dict["items"][0]["value"].get("tool_calls") + if raw_tool_calls: + additional_kwargs["tool_calls"] = raw_tool_calls + for raw_tool_call in raw_tool_calls: + if isinstance(raw_tool_call["function"]["arguments"], dict): + raw_tool_call["function"]["arguments"] = json.dumps( + raw_tool_call["function"].get("arguments", {}) + ) + try: + tool_calls.append( + parse_tool_call(raw_tool_call, return_id=True) + ) + except Exception as e: + invalid_tool_calls.append( + make_invalid_tool_call(raw_tool_call, str(e)) + ) # process response payload for generic v1 API elif "api/predict/generic" in self.sambastudio_url: @@ -1064,7 +1177,9 @@ def _process_response(self, response: Response) -> AIMessage: return AIMessage( content=content, - additional_kwargs={}, + additional_kwargs=additional_kwargs, + tool_calls=tool_calls, + invalid_tool_calls=invalid_tool_calls, response_metadata=response_metadata, id=id, ) @@ -1307,7 +1422,7 @@ def _generate( ) if stream_iter: return generate_from_stream(stream_iter) - response = self._handle_request(messages, stop, streaming=False) + response = self._handle_request(messages, stop, streaming=False, **kwargs) message = self._process_response(response) generation = ChatGeneration(message=message) return ChatResult(generations=[generation]) @@ -1335,7 +1450,7 @@ def _stream( Yields: chunk: ChatGenerationChunk with model partial generation """ - response = self._handle_request(messages, stop, streaming=True) + response = self._handle_request(messages, stop, streaming=True, **kwargs) for ai_message_chunk in self._process_stream_response(response): chunk = ChatGenerationChunk(message=ai_message_chunk) if run_manager: From f7731ba87aa6cefa5fc23cbebaefd4cf1af2a729 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Mon, 11 Nov 2024 11:03:48 -0500 Subject: [PATCH 08/30] add tool calling to samabstudio chat model docs --- docs/docs/integrations/chat/sambastudio.ipynb | 97 ++++++++++++++++++- 1 file changed, 93 insertions(+), 4 deletions(-) diff --git a/docs/docs/integrations/chat/sambastudio.ipynb b/docs/docs/integrations/chat/sambastudio.ipynb index 64dd05fd96b8c..e719354038fb0 100644 --- a/docs/docs/integrations/chat/sambastudio.ipynb +++ b/docs/docs/integrations/chat/sambastudio.ipynb @@ -34,7 +34,7 @@ "\n", "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", + "| ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", "\n", "## Setup\n", "\n", @@ -119,20 +119,20 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from langchain_community.chat_models.sambanova import ChatSambaStudio\n", "\n", "llm = ChatSambaStudio(\n", - " model=\"Meta-Llama-3-70B-Instruct-4096\", # set if using a CoE endpoint\n", + " model=\"Meta-Llama-3-70B-Instruct-4096\", # set if using a Bundle endpoint\n", " max_tokens=1024,\n", " temperature=0.7,\n", " top_k=1,\n", " top_p=0.01,\n", " do_sample=True,\n", - " process_prompt=\"True\", # set if using a CoE endpoint\n", + " process_prompt=\"True\", # set if using a Bundle endpoint\n", ")" ] }, @@ -349,6 +349,95 @@ " print(chunk.content, end=\"\", flush=True)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool calling" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage\n", + "from langchain_core.tools import tool\n", + "\n", + "\n", + "@tool\n", + "def get_time(kind: str = \"both\") -> str:\n", + " \"\"\"Returns current date, current time or both.\n", + " Args:\n", + " kind: date, time or both\n", + " \"\"\"\n", + " if kind == \"date\":\n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " return f\"Current date: {date}\"\n", + " elif kind == \"time\":\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current time: {time}\"\n", + " else:\n", + " date = datetime.now().strftime(\"%m/%d/%Y\")\n", + " time = datetime.now().strftime(\"%H:%M:%S\")\n", + " return f\"Current date: {date}, Current time: {time}\"\n", + "\n", + "\n", + "def invoke_tools(tool_calls, messages):\n", + " for tool_call in tool_calls:\n", + " selected_tool = {\"get_time\": get_time}[tool_call[\"name\"].lower()]\n", + " tool_output = selected_tool.invoke(tool_call[\"args\"])\n", + " print(f\"Tool output: {tool_output}\")\n", + " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", + " return messages\n", + "\n", + "\n", + "tools = [get_time]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "llm_with_tools = llm.bind_tools(tools=tools)\n", + "messages = [\n", + " HumanMessage(\n", + " content=\"I need to schedule a meeting for two weeks from today. Can you tell me the exact date of the meeting?\"\n", + " )\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Intermediate model response: [{'name': 'get_time', 'args': {'kind': 'date'}, 'id': 'call_4092d5dd21cd4eb494', 'type': 'tool_call'}]\n", + "Tool output: Current date: 11/07/2024\n", + "final response: The meeting will be exactly two weeks from today, which would be 25/07/2024.\n" + ] + } + ], + "source": [ + "response = llm_with_tools.invoke(messages)\n", + "if response.tool_calls:\n", + " print(f\"Intermediate model response: {response.tool_calls}\")\n", + " messages.append(response)\n", + " messages = invoke_tools(response.tool_calls, messages)\n", + "response = llm.invoke(messages)\n", + "\n", + "print(f\"final response: {response.content}\")" + ] + }, { "cell_type": "markdown", "metadata": {}, From 759836bd2c096bc4f28cc1a316f1c1375aa8cfc8 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Mon, 11 Nov 2024 11:06:57 -0500 Subject: [PATCH 09/30] minor renaming CoE convention to bundle --- .../chat_models/sambanova.py | 24 +++++++++---------- .../langchain_community/llms/sambanova.py | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 6df9c07844fb6..afdc9dd0308e5 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -599,17 +599,17 @@ class ChatSambaStudio(BaseChatModel): ChatSambaStudio( sambastudio_url = set with your SambaStudio deployed endpoint URL, sambastudio_api_key = set with your SambaStudio deployed endpoint Key. - model = model or expert name (set for CoE endpoints), + model = model or expert name (set for Bundle endpoints), max_tokens = max number of tokens to generate, temperature = model temperature, top_p = model top p, top_k = model top k, do_sample = wether to do sample process_prompt = wether to process prompt - (set for CoE generic v1 and v2 endpoints) + (set for Bundle generic v1 and v2 endpoints) stream_options = include usage to get generation metrics special_tokens = start, start_role, end_role, end special tokens - (set for CoE generic v1 and v2 endpoints when process prompt + (set for Bundle generic v1 and v2 endpoints when process prompt set to false or for StandAlone v1 and v2 endpoints) model_kwargs: Optional = Extra Key word arguments to pass to the model. ) @@ -617,7 +617,7 @@ class ChatSambaStudio(BaseChatModel): Key init args — completion params: model: str The name of the model to use, e.g., Meta-Llama-3-70B-Instruct-4096 - (set for CoE endpoints). + (set for Bundle endpoints). streaming: bool Whether to use streaming max_tokens: inthandler when using non streaming methods @@ -631,12 +631,12 @@ class ChatSambaStudio(BaseChatModel): do_sample: bool wether to do sample process_prompt: - wether to process prompt (set for CoE generic v1 and v2 endpoints) + wether to process prompt (set for Bundle generic v1 and v2 endpoints) stream_options: dict stream options, include usage to get generation metrics special_tokens: dict start, start_role, end_role and end special tokens - (set for CoE generic v1 and v2 endpoints when process prompt set to false + (set for Bundle generic v1 and v2 endpoints when process prompt set to false or for StandAlone v1 and v2 endpoints) default to llama3 special tokens model_kwargs: dict Extra Key word arguments to pass to the model. @@ -655,17 +655,17 @@ class ChatSambaStudio(BaseChatModel): chat = ChatSambaStudio=( sambastudio_url = set with your SambaStudio deployed endpoint URL, sambastudio_api_key = set with your SambaStudio deployed endpoint Key. - model = model or expert name (set for CoE endpoints), + model = model or expert name (set for Bundle endpoints), max_tokens = max number of tokens to generate, temperature = model temperature, top_p = model top p, top_k = model top k, do_sample = wether to do sample process_prompt = wether to process prompt - (set for CoE generic v1 and v2 endpoints) + (set for Bundle generic v1 and v2 endpoints) stream_options = include usage to get generation metrics special_tokens = start, start_role, end_role, and special tokens - (set for CoE generic v1 and v2 endpoints when process prompt + (set for Bundle generic v1 and v2 endpoints when process prompt set to false or for StandAlone v1 and v2 endpoints) model_kwargs: Optional = Extra Key word arguments to pass to the model. ) @@ -742,7 +742,7 @@ class GetWeather(BaseModel): """SambaStudio streaming Url""" model: Optional[str] = Field(default=None) - """The name of the model or expert to use (for CoE endpoints)""" + """The name of the model or expert to use (for Bundle endpoints)""" streaming: bool = Field(default=False) """Whether to use streaming handler when using non streaming methods""" @@ -763,7 +763,7 @@ class GetWeather(BaseModel): """whether to do sampling""" process_prompt: Optional[bool] = Field(default=True) - """whether process prompt (for CoE generic v1 and v2 endpoints)""" + """whether process prompt (for Bundle generic v1 and v2 endpoints)""" stream_options: Dict[str, Any] = Field(default={"include_usage": True}) """stream options, include usage to get generation metrics""" @@ -777,7 +777,7 @@ class GetWeather(BaseModel): } ) """start, start_role, end_role and end special tokens - (set for CoE generic v1 and v2 endpoints when process prompt set to false + (set for Bundle generic v1 and v2 endpoints when process prompt set to false or for StandAlone v1 and v2 endpoints) default to llama3 special tokens""" diff --git a/libs/community/langchain_community/llms/sambanova.py b/libs/community/langchain_community/llms/sambanova.py index 4ae6a1e6a2809..3feed75e15ea7 100644 --- a/libs/community/langchain_community/llms/sambanova.py +++ b/libs/community/langchain_community/llms/sambanova.py @@ -27,20 +27,20 @@ class SambaStudio(LLM): sambastudio_url="your-SambaStudio-environment-URL", sambastudio_api_key="your-SambaStudio-API-key, model_kwargs={ - "model" : model or expert name (set for CoE endpoints), + "model" : model or expert name (set for Bundle endpoints), "max_tokens" : max number of tokens to generate, "temperature" : model temperature, "top_p" : model top p, "top_k" : model top k, "do_sample" : wether to do sample "process_prompt": wether to process prompt - (set for CoE generic v1 and v2 endpoints) + (set for Bundle generic v1 and v2 endpoints) }, ) Key init args — completion params: model: str The name of the model to use, e.g., Meta-Llama-3-70B-Instruct-4096 - (set for CoE endpoints). + (set for Bundle endpoints). streaming: bool Whether to use streaming handler when using non streaming methods model_kwargs: dict @@ -56,7 +56,7 @@ class SambaStudio(LLM): do_sample: bool wether to do sample process_prompt: - wether to process prompt (set for CoE generic v1 and v2 endpoints) + wether to process prompt (set for Bundle generic v1 and v2 endpoints) Key init args — client params: sambastudio_url: str SambaStudio endpoint Url @@ -72,14 +72,14 @@ class SambaStudio(LLM): sambastudio_url = set with your SambaStudio deployed endpoint URL, sambastudio_api_key = set with your SambaStudio deployed endpoint Key, model_kwargs = { - "model" : model or expert name (set for CoE endpoints), + "model" : model or expert name (set for Bundle endpoints), "max_tokens" : max number of tokens to generate, "temperature" : model temperature, "top_p" : model top p, "top_k" : model top k, "do_sample" : wether to do sample "process_prompt" : wether to process prompt - (set for CoE generic v1 and v2 endpoints) + (set for Bundle generic v1 and v2 endpoints) } ) From 89f5ae50059d9933daf5b1b0c36e219dd030250e Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Mon, 11 Nov 2024 11:14:42 -0500 Subject: [PATCH 10/30] fmt --- libs/community/langchain_community/llms/sambanova.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/community/langchain_community/llms/sambanova.py b/libs/community/langchain_community/llms/sambanova.py index 3feed75e15ea7..0979dabd13398 100644 --- a/libs/community/langchain_community/llms/sambanova.py +++ b/libs/community/langchain_community/llms/sambanova.py @@ -56,7 +56,8 @@ class SambaStudio(LLM): do_sample: bool wether to do sample process_prompt: - wether to process prompt (set for Bundle generic v1 and v2 endpoints) + wether to process prompt + (set for Bundle generic v1 and v2 endpoints) Key init args — client params: sambastudio_url: str SambaStudio endpoint Url From b3d87fba83183cd0594906e83c6086e1f62508bb Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Mon, 11 Nov 2024 11:20:10 -0500 Subject: [PATCH 11/30] fmt --- libs/community/langchain_community/llms/sambanova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/llms/sambanova.py b/libs/community/langchain_community/llms/sambanova.py index 0979dabd13398..41dfabd04c251 100644 --- a/libs/community/langchain_community/llms/sambanova.py +++ b/libs/community/langchain_community/llms/sambanova.py @@ -56,7 +56,7 @@ class SambaStudio(LLM): do_sample: bool wether to do sample process_prompt: - wether to process prompt + wether to process prompt (set for Bundle generic v1 and v2 endpoints) Key init args — client params: sambastudio_url: str From e3c8e0b06fd635f9c44ae8d52abedc72d2a4fcaf Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 12:42:39 -0500 Subject: [PATCH 12/30] minor change in mesagges dicts --- .../langchain_community/chat_models/sambanova.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index ff5ddd3f2e5ad..3264f431eef41 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -54,6 +54,7 @@ def _convert_message_to_dict(message: BaseMessage) -> Dict[str, Any]: Returns: messages_dict: role / content dict """ + message_dict: Dict[str, Any] = {} if isinstance(message, ChatMessage): message_dict = {"role": message.role, "content": message.content} elif isinstance(message, SystemMessage): @@ -62,8 +63,16 @@ def _convert_message_to_dict(message: BaseMessage) -> Dict[str, Any]: message_dict = {"role": "user", "content": message.content} elif isinstance(message, AIMessage): message_dict = {"role": "assistant", "content": message.content} + if "tool_calls" in message.additional_kwargs: + message_dict["tool_calls"] = message.additional_kwargs["tool_calls"] + if message_dict["content"] == "": + message_dict["content"] = None elif isinstance(message, ToolMessage): - message_dict = {"role": "tool", "content": message.content} + message_dict = { + "role": "tool", + "content": message.content, + "tool_call_id": message.tool_call_id, + } else: raise TypeError(f"Got unknown type {message}") return message_dict @@ -229,6 +238,9 @@ class GetWeather(BaseModel): stream_options: Dict[str, Any] = Field(default={"include_usage": True}) """stream options, include usage to get generation metrics""" + additional_headers: Dict[str, Any] = Field(default={}) + """Additional headers to sent in request""" + class Config: populate_by_name = True @@ -363,6 +375,7 @@ def _handle_request( headers={ "Authorization": f"Bearer {self.sambanova_api_key.get_secret_value()}", "Content-Type": "application/json", + **self.additional_headers, }, json=data, stream=streaming, From 174707222831f8fe35e4ae23b02f98dc95ab88ca Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 12:47:11 -0500 Subject: [PATCH 13/30] minor changes --- docs/docs/integrations/chat/sambanova.ipynb | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index c41d37cfa67df..7a30efb4e072b 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -357,7 +357,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -385,16 +385,17 @@ " return f\"Current date: {date}, Current time: {time}\"\n", "\n", "\n", + "tools = [get_time]\n", + "\n", + "\n", "def invoke_tools(tool_calls, messages):\n", + " available_functions = {tool.name: tool for tool in tools}\n", " for tool_call in tool_calls:\n", - " selected_tool = {\"get_time\": get_time}[tool_call[\"name\"].lower()]\n", + " selected_tool = available_functions[tool_call[\"name\"]]\n", " tool_output = selected_tool.invoke(tool_call[\"args\"])\n", " print(f\"Tool output: {tool_output}\")\n", " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", - " return messages\n", - "\n", - "\n", - "tools = [get_time]" + " return messages" ] }, { @@ -413,7 +414,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -428,11 +429,11 @@ ], "source": [ "response = llm_with_tools.invoke(messages)\n", - "if response.tool_calls:\n", + "while len(response.tool_calls) > 0:\n", " print(f\"Intermediate model response: {response.tool_calls}\")\n", " messages.append(response)\n", " messages = invoke_tools(response.tool_calls, messages)\n", - "response = llm.invoke(messages)\n", + "response = llm_with_tools.invoke(messages)\n", "\n", "print(f\"final response: {response.content}\")" ] From 1be98841f3cddcb27f9987bddee864538114a049 Mon Sep 17 00:00:00 2001 From: Jorge Piedrahita Ortiz Date: Wed, 13 Nov 2024 12:52:39 -0500 Subject: [PATCH 14/30] fix typo --- libs/community/langchain_community/chat_models/sambanova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 3264f431eef41..414f90e99965f 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -239,7 +239,7 @@ class GetWeather(BaseModel): """stream options, include usage to get generation metrics""" additional_headers: Dict[str, Any] = Field(default={}) - """Additional headers to sent in request""" + """Additional headers to send in request""" class Config: populate_by_name = True From 43ae94d99e51c92c353a07eacf6e1024914059b8 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 13:03:22 -0500 Subject: [PATCH 15/30] minor changes --- docs/docs/integrations/chat/sambastudio.ipynb | 15 ++++++++------- .../langchain_community/chat_models/sambanova.py | 9 +++++++-- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/docs/integrations/chat/sambastudio.ipynb b/docs/docs/integrations/chat/sambastudio.ipynb index e719354038fb0..dcf1c450c55c6 100644 --- a/docs/docs/integrations/chat/sambastudio.ipynb +++ b/docs/docs/integrations/chat/sambastudio.ipynb @@ -386,16 +386,17 @@ " return f\"Current date: {date}, Current time: {time}\"\n", "\n", "\n", + "tools = [get_time]\n", + "\n", + "\n", "def invoke_tools(tool_calls, messages):\n", + " available_functions = {tool.name: tool for tool in tools}\n", " for tool_call in tool_calls:\n", - " selected_tool = {\"get_time\": get_time}[tool_call[\"name\"].lower()]\n", + " selected_tool = available_functions[tool_call[\"name\"]]\n", " tool_output = selected_tool.invoke(tool_call[\"args\"])\n", " print(f\"Tool output: {tool_output}\")\n", " messages.append(ToolMessage(tool_output, tool_call_id=tool_call[\"id\"]))\n", - " return messages\n", - "\n", - "\n", - "tools = [get_time]" + " return messages" ] }, { @@ -429,11 +430,11 @@ ], "source": [ "response = llm_with_tools.invoke(messages)\n", - "if response.tool_calls:\n", + "while len(response.tool_calls) > 0:\n", " print(f\"Intermediate model response: {response.tool_calls}\")\n", " messages.append(response)\n", " messages = invoke_tools(response.tool_calls, messages)\n", - "response = llm.invoke(messages)\n", + "response = llm_with_tools.invoke(messages)\n", "\n", "print(f\"final response: {response.content}\")" ] diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 0227a6a6e2e81..0ffabb9e857ec 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -796,6 +796,9 @@ class GetWeather(BaseModel): model_kwargs: Optional[Dict[str, Any]] = None """Key word arguments to pass to the model.""" + + additional_headers: Dict[str, Any] = Field(default={}) + """Additional headers to send in request""" class Config: populate_by_name = True @@ -942,6 +945,7 @@ def _messages_to_string(self, messages: List[BaseMessage]) -> str: "content": message.content, } ) + #TODO add tools msgs id and assistant msgs tool calls messages_string = json.dumps(messages_dict) else: messages_string = self.special_tokens["start"] @@ -1020,6 +1024,7 @@ def _handle_request( "Authorization": f"Bearer " f"{self.sambastudio_api_key.get_secret_value()}", "Content-Type": "application/json", + **self.additional_headers } # create request payload for generic v1 API @@ -1039,7 +1044,7 @@ def _handle_request( params = {**params, **self.model_kwargs} params = {key: value for key, value in params.items() if value is not None} data = {"items": items, "params": params} - headers = {"key": self.sambastudio_api_key.get_secret_value()} + headers = {"key": self.sambastudio_api_key.get_secret_value(), **self.additional_headers} # create request payload for generic v1 API elif "api/predict/generic" in self.sambastudio_url: @@ -1075,7 +1080,7 @@ def _handle_request( "instances": [self._messages_to_string(messages)], "params": params, } - headers = {"key": self.sambastudio_api_key.get_secret_value()} + headers = {"key": self.sambastudio_api_key.get_secret_value(), **self.additional_headers} else: raise ValueError( From 7b53800e27686e17793de50ad01261c6ddba89be Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 13:18:50 -0500 Subject: [PATCH 16/30] fmt --- .../langchain_community/chat_models/sambanova.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 0ffabb9e857ec..27b042173ad76 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -796,7 +796,7 @@ class GetWeather(BaseModel): model_kwargs: Optional[Dict[str, Any]] = None """Key word arguments to pass to the model.""" - + additional_headers: Dict[str, Any] = Field(default={}) """Additional headers to send in request""" @@ -945,7 +945,7 @@ def _messages_to_string(self, messages: List[BaseMessage]) -> str: "content": message.content, } ) - #TODO add tools msgs id and assistant msgs tool calls + # TODO add tools msgs id and assistant msgs tool calls messages_string = json.dumps(messages_dict) else: messages_string = self.special_tokens["start"] @@ -1024,7 +1024,7 @@ def _handle_request( "Authorization": f"Bearer " f"{self.sambastudio_api_key.get_secret_value()}", "Content-Type": "application/json", - **self.additional_headers + **self.additional_headers, } # create request payload for generic v1 API @@ -1044,7 +1044,10 @@ def _handle_request( params = {**params, **self.model_kwargs} params = {key: value for key, value in params.items() if value is not None} data = {"items": items, "params": params} - headers = {"key": self.sambastudio_api_key.get_secret_value(), **self.additional_headers} + headers = { + "key": self.sambastudio_api_key.get_secret_value(), + **self.additional_headers, + } # create request payload for generic v1 API elif "api/predict/generic" in self.sambastudio_url: @@ -1080,7 +1083,10 @@ def _handle_request( "instances": [self._messages_to_string(messages)], "params": params, } - headers = {"key": self.sambastudio_api_key.get_secret_value(), **self.additional_headers} + headers = { + "key": self.sambastudio_api_key.get_secret_value(), + **self.additional_headers, + } else: raise ValueError( From 2e1b46baf4508a2d28f98b5d32bb363f08aecfac Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:05:14 -0500 Subject: [PATCH 17/30] add structured output --- .../chat_models/sambanova.py | 354 +++++++++++++++++- 1 file changed, 351 insertions(+), 3 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 414f90e99965f..bfceb052ac8a9 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -1,15 +1,18 @@ import json +from operator import itemgetter from typing import ( Any, Callable, Dict, Iterator, List, + Literal, Optional, Sequence, Tuple, Type, Union, + cast, ) import requests @@ -31,16 +34,24 @@ SystemMessage, ToolMessage, ) +from langchain_core.output_parsers import ( + JsonOutputParser, + PydanticOutputParser, +) +from langchain_core.output_parsers.base import OutputParserLike from langchain_core.output_parsers.openai_tools import ( + JsonOutputKeyToolsParser, + PydanticToolsParser, make_invalid_tool_call, parse_tool_call, ) from langchain_core.outputs import ChatGeneration, ChatGenerationChunk, ChatResult -from langchain_core.runnables import Runnable +from langchain_core.runnables import Runnable, RunnableMap, RunnablePassthrough from langchain_core.tools import BaseTool from langchain_core.utils import convert_to_secret_str, get_from_dict_or_env from langchain_core.utils.function_calling import convert_to_openai_tool -from pydantic import Field, SecretStr +from langchain_core.utils.pydantic import is_basemodel_subclass +from pydantic import BaseModel, Field, SecretStr from requests import Response @@ -92,6 +103,10 @@ def _create_message_dicts(messages: List[BaseMessage]) -> List[Dict[str, Any]]: return message_dicts +def _is_pydantic_class(obj: Any) -> bool: + return isinstance(obj, type) and is_basemodel_subclass(obj) + + class ChatSambaNovaCloud(BaseChatModel): """ SambaNova Cloud chat model. @@ -198,6 +213,29 @@ class GetWeather(BaseModel): } ] + Structured output: + .. code-block:: python + + from typing import Optional + + from pydantic import BaseModel, Field + + class Joke(BaseModel): + '''Joke to tell user.''' + + setup: str = Field(description="The setup of the joke") + punchline: str = Field(description="The punchline to the joke") + + structured_model = llm.with_structured_output(Joke) + structured_model.invoke("Tell me a joke about cats") + + .. code-block:: python + + Joke(setup="Why did the cat join a band?", + punchline="Because it wanted to be the purr-cussionist!") + + See ``ChatSambanovaCloud.with_structured_output()`` for more. + Token usage: .. code-block:: python response = chat.invoke(messages) @@ -239,7 +277,7 @@ class GetWeather(BaseModel): """stream options, include usage to get generation metrics""" additional_headers: Dict[str, Any] = Field(default={}) - """Additional headers to send in request""" + """Additional headers to sent in request""" class Config: populate_by_name = True @@ -327,6 +365,316 @@ def bind_tools( kwargs["parallel_tool_calls"] = parallel_tool_calls return super().bind(tools=formatted_tools, **kwargs) + def with_structured_output( + self, + schema: Optional[Union[Dict[str, Any], Type[BaseModel]]] = None, + *, + method: Literal[ + "function_calling", "json_mode", "json_schema" + ] = "function_calling", + include_raw: bool = False, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, Union[Dict[str, Any], BaseModel]]: + """Model wrapper that returns outputs formatted to match the given schema. + + Args: + schema: + The output schema. Can be passed in as: + - an OpenAI function/tool schema, + - a JSON Schema, + - a TypedDict class, + - or a Pydantic class. + If ``schema`` is a Pydantic class then the model output will be a + Pydantic instance of that class, and the model-generated fields will be + validated by the Pydantic class. Otherwise the model output will be a + dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` + for more on how to properly specify types and descriptions of + schema fields when specifying a Pydantic or TypedDict class. + + method: + The method for steering model generation, either "function_calling" + "json_mode" or "json_schema". + If "function_calling" then the schema will be converted + to an OpenAI function and the returned model will make use of the + function-calling API. If "json_mode" or "json_schema" then OpenAI's + JSON mode will be used. + Note that if using "json_mode" or "json_schema" then you must include instructions + for formatting the output into the desired schema into the model call. + + include_raw: + If False then only the parsed structured output is returned. If + an error occurs during model output parsing it will be raised. If True + then both the raw model response (a BaseMessage) and the parsed model + response will be returned. If an error occurs during output parsing it + will be caught and returned as well. The final output is always a dict + with keys "raw", "parsed", and "parsing_error". + + Returns: + A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. + + If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs + an instance of ``schema`` (i.e., a Pydantic object). + + Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + + If ``include_raw`` is True, then Runnable outputs a dict with keys: + - ``"raw"``: BaseMessage + - ``"parsed"``: None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. + - ``"parsing_error"``: Optional[BaseException] + + Example: schema=Pydantic class, method="function_calling", include_raw=False: + .. code-block:: python + + from typing import Optional + + from langchain_community.chat_models import ChatSambaNovaCloud + from pydantic import BaseModel, Field + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str = Field( + description="A justification for the answer." + ) + + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + + # -> AnswerWithJustification( + # answer='They weigh the same', + # justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same.' + # ) + + Example: schema=Pydantic class, method="function_calling", include_raw=True: + .. code-block:: python + + from langchain_community.chat_models import ChatSambaNovaCloud + from pydantic import BaseModel + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str + + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, include_raw=True + ) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'function': {'arguments': '{"answer": "They weigh the same.", "justification": "A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount."}', 'name': 'AnswerWithJustification'}, 'id': 'call_17a431fc6a4240e1bd', 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls', 'usage': {'acceptance_rate': 5, 'completion_tokens': 53, 'completion_tokens_after_first_per_sec': 343.7964936837758, 'completion_tokens_after_first_per_sec_first_ten': 439.1205661878638, 'completion_tokens_per_sec': 162.8511306784833, 'end_time': 1731527851.0698032, 'is_last_response': True, 'prompt_tokens': 213, 'start_time': 1731527850.7137961, 'time_to_first_token': 0.20475482940673828, 'total_latency': 0.32545061111450196, 'total_tokens': 266, 'total_tokens_per_sec': 817.3283162354066}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731527850}, id='95667eaf-447f-4b53-bb6e-b6e1094ded88', tool_calls=[{'name': 'AnswerWithJustification', 'args': {'answer': 'They weigh the same.', 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.'}, 'id': 'call_17a431fc6a4240e1bd', 'type': 'tool_call'}]), + # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.'), + # 'parsing_error': None + # } + + Example: schema=TypedDict class, method="function_calling", include_raw=False: + .. code-block:: python + + # IMPORTANT: If you are using Python <=3.8, you need to import Annotated + # from typing_extensions, not from typing. + from typing_extensions import Annotated, TypedDict + + from langchain_community.chat_models import ChatSambaNovaCloud + + + class AnswerWithJustification(TypedDict): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Annotated[ + Optional[str], None, "A justification for the answer." + ] + + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.' + # } + + Example: schema=OpenAI function schema, method="function_calling", include_raw=False: + .. code-block:: python + + from langchain_community.chat_models import ChatSambaNovaCloud + + oai_schema = { + 'name': 'AnswerWithJustification', + 'description': 'An answer to the user question along with justification for the answer.', + 'parameters': { + 'type': 'object', + 'properties': { + 'answer': {'type': 'string'}, + 'justification': {'description': 'A justification for the answer.', 'type': 'string'} + }, + 'required': ['answer'] + } + } + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(oai_schema) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.' + # } + + Example: schema=Pydantic class, method="json_mode", include_raw=True: + .. code-block:: + + from langchain_community.chat_models import ChatSambaNovaCloud + from pydantic import BaseModel + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, + method="json_mode", + include_raw=True + ) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 5.3125, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 292.65701089829776, 'completion_tokens_after_first_per_sec_first_ten': 346.43324678555325, 'completion_tokens_per_sec': 200.012158915008, 'end_time': 1731528071.1708555, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528070.737394, 'time_to_first_token': 0.16693782806396484, 'total_latency': 0.3949759876026827, 'total_tokens': 149, 'total_tokens_per_sec': 377.2381225105847}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528070}, id='83208297-3eb9-4021-a856-ca78a15758df'), + # 'parsed': AnswerWithJustification(answer='They are the same weight', justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'), + # 'parsing_error': None + # } + + Example: schema=None, method="json_mode", include_raw=True: + .. code-block:: + from langchain_community.chat_models import ChatSambaNovaCloud + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(method="json_mode", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 4.722222222222222, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 357.1315485254867, 'completion_tokens_after_first_per_sec_first_ten': 416.83279609305305, 'completion_tokens_per_sec': 240.92819585198137, 'end_time': 1731528164.8474727, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528164.4906917, 'time_to_first_token': 0.13837409019470215, 'total_latency': 0.3278985247892492, 'total_tokens': 149, 'total_tokens_per_sec': 454.4088757208256}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528164}, id='15261eaf-8a25-42ef-8ed5-f63d8bf5b1b0'), + # 'parsed': { + # 'answer': 'They are the same weight', + # 'justification': 'A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'}, + # }, + # 'parsing_error': None + # } + + Example: schema=None, method="json_schema", include_raw=True: + .. code-block:: + from langchain_community.chat_models import ChatSambaNovaCloud + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification, method="json_schema", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 5.3125, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 292.65701089829776, 'completion_tokens_after_first_per_sec_first_ten': 346.43324678555325, 'completion_tokens_per_sec': 200.012158915008, 'end_time': 1731528071.1708555, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528070.737394, 'time_to_first_token': 0.16693782806396484, 'total_latency': 0.3949759876026827, 'total_tokens': 149, 'total_tokens_per_sec': 377.2381225105847}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528070}, id='83208297-3eb9-4021-a856-ca78a15758df'), + # 'parsed': AnswerWithJustification(answer='They are the same weight', justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'), + # 'parsing_error': None + # } + """ # noqa: E501 + if kwargs: + raise ValueError(f"Received unsupported arguments {kwargs}") + is_pydantic_schema = _is_pydantic_class(schema) + if method == "function_calling": + if schema is None: + raise ValueError( + "schema must be specified when method is 'function_calling'. " + "Received None." + ) + tool_name = convert_to_openai_tool(schema)["function"]["name"] + llm = self.bind_tools([schema], tool_choice=tool_name) + if is_pydantic_schema: + output_parser: OutputParserLike[Any] = PydanticToolsParser( + tools=[schema], # type: ignore[list-item] + first_tool_only=True, + ) + else: + output_parser = JsonOutputKeyToolsParser( + key_name=tool_name, first_tool_only=True + ) + elif method == "json_mode": + llm = self + # TODO bind response format when json mode available by API + # llm = self.bind(response_format={"type": "json_object"}) + if is_pydantic_schema: + schema = cast(Type[BaseModel], schema) + output_parser = PydanticOutputParser(pydantic_object=schema) + else: + output_parser = JsonOutputParser() + + elif method == "json_schema": + if schema is None: + raise ValueError( + "schema must be specified when method is not 'json_mode'. " + "Received None." + ) + llm = self + # TODO bind response format when json schema available by API, + # update example + # llm = self.bind( + # response_format={"type": "json_object", "json_schema": schema} + # ) + if is_pydantic_schema: + schema = cast(Type[BaseModel], schema) + output_parser = PydanticOutputParser(pydantic_object=schema) + else: + output_parser = JsonOutputParser() + else: + raise ValueError( + f"Unrecognized method argument. Expected one of 'function_calling' or " + f"'json_mode'. Received: '{method}'" + ) + + if include_raw: + parser_assign = RunnablePassthrough.assign( + parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None + ) + parser_none = RunnablePassthrough.assign(parsed=lambda _: None) + parser_with_fallback = parser_assign.with_fallbacks( + [parser_none], exception_key="parsing_error" + ) + return RunnableMap(raw=llm) | parser_with_fallback + else: + return llm | output_parser + def _handle_request( self, messages_dicts: List[Dict[str, Any]], From c9a87a121bb7a4da94e40d774cf86d210b01ba0e Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:05:54 -0500 Subject: [PATCH 18/30] add structured output and image examples to integrations docs --- docs/docs/integrations/chat/sambanova.ipynb | 155 ++++++++++++++++---- 1 file changed, 129 insertions(+), 26 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index 7a30efb4e072b..f81a6c2d02b05 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -34,7 +34,7 @@ "\n", "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", + "| ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", "\n", "## Setup\n", "\n", @@ -116,11 +116,12 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ - "from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", + "# from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", + "from langchain_chat_models import ChatSambaNovaCloud\n", "\n", "llm = ChatSambaNovaCloud(\n", " model=\"Meta-Llama-3.1-70B-Instruct\",\n", @@ -146,7 +147,7 @@ { "data": { "text/plain": [ - "AIMessage(content=\"J'adore la programmation.\", additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 14, 'completion_tokens': 8, 'completion_tokens_after_first_per_sec': 164.48250980392157, 'completion_tokens_after_first_per_sec_first_ten': 854.0507017671442, 'completion_tokens_per_sec': 33.17909116130268, 'end_time': 1731001001.274775, 'is_last_response': True, 'prompt_tokens': 55, 'start_time': 1731001001.0004687, 'time_to_first_token': 0.2317485809326172, 'total_latency': 0.24111570630754744, 'total_tokens': 63, 'total_tokens_per_sec': 261.2853428952586}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001000}, id='415968fc-1a58-47e0-a8a3-7cb80383980b')" + "AIMessage(content=\"J'adore la programmation.\", additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 7, 'completion_tokens': 8, 'completion_tokens_after_first_per_sec': 195.0204119588971, 'completion_tokens_after_first_per_sec_first_ten': 618.3422770734173, 'completion_tokens_per_sec': 53.25837044790076, 'end_time': 1731535338.1864908, 'is_last_response': True, 'prompt_tokens': 55, 'start_time': 1731535338.0133238, 'time_to_first_token': 0.13727331161499023, 'total_latency': 0.15021112986973353, 'total_tokens': 63, 'total_tokens_per_sec': 419.4096672772185}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731535338}, id='f04b7c2c-bc46-47e0-9c6b-19a002e8f390')" ] }, "execution_count": 3, @@ -200,7 +201,7 @@ { "data": { "text/plain": [ - "AIMessage(content='Ich liebe das Programmieren.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 2.3333333333333335, 'completion_tokens': 6, 'completion_tokens_after_first_per_sec': 100.67263842084546, 'completion_tokens_after_first_per_sec_first_ten': 205.99550965424336, 'completion_tokens_per_sec': 36.087854367623336, 'end_time': 1731001005.4113064, 'is_last_response': True, 'prompt_tokens': 50, 'start_time': 1731001005.2245064, 'time_to_first_token': 0.13713407516479492, 'total_latency': 0.1662609236580985, 'total_tokens': 56, 'total_tokens_per_sec': 336.8199740978178}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001005}, id='0990bec1-1721-41f9-a14c-d0bea8ac6263')" + "AIMessage(content='Ich liebe das Programmieren.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 2.3333333333333335, 'completion_tokens': 6, 'completion_tokens_after_first_per_sec': 106.06729752831038, 'completion_tokens_after_first_per_sec_first_ten': 204.92722183833433, 'completion_tokens_per_sec': 26.32497272023831, 'end_time': 1731535339.9997504, 'is_last_response': True, 'prompt_tokens': 50, 'start_time': 1731535339.7539687, 'time_to_first_token': 0.19864177703857422, 'total_latency': 0.22792046410696848, 'total_tokens': 56, 'total_tokens_per_sec': 245.6997453888909}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731535339}, id='dfe0bee6-b297-472e-ac9d-29906d162dcb')" ] }, "execution_count": 5, @@ -247,19 +248,24 @@ "name": "stdout", "output_type": "stream", "text": [ - "Yer lookin' fer some knowledge about owls, eh? Alright then, matey, settle yerself down with a pint o' grog and listen close.\n", + "Yer lookin' fer some knowledge about owls, eh? Alright then, matey, settle yerself down with a pint o' grog and listen close. \n", "\n", "Owls be a fascinatin' lot, with their big round eyes and silent wings. They be birds o' prey, which means they hunt other creatures fer food. There be over 220 species o' owls, rangin' in size from the tiny Elf Owl (which be smaller than a parrot) to the Great Grey Owl (which be as big as a small eagle).\n", "\n", - "Here be some o' the most interestin' facts about owls:\n", + "One o' the most interestin' things about owls be their eyes. They be huge, with some species havin' eyes that be as big as their brains! This lets 'em see in the dark, which be perfect fer nocturnal huntin'. They also have special feathers on their faces that help 'em hear better, and their ears be specially designed to pinpoint sounds.\n", + "\n", + "Owls be known fer their silent flight, which be due to the special shape o' their wings. They be able to fly without makin' a sound, which be perfect fer sneakin' up on prey. They also be very agile, with some species able to fly through tight spaces and make sharp turns.\n", + "\n", + "Some o' the most common species o' owls include:\n", "\n", - "1. **Night vision**: Owls have eyes that be specially designed fer seein' in the dark. Their eyes be big and round, with a reflective layer called the tapetum lucidum that helps 'em see in low light.\n", - "2. **Silent flyers**: Owls have special feathers on their wings that help 'em fly silently. This be useful fer sneakin' up on prey, matey!\n", - "3. **Turnin' heads**: Owls can turn their heads up to 270 degrees, which be useful fer seein' what be goin' on around 'em without havin' to move their whole body.\n", - "4. **Huntin' skills**: Owls be skilled hunters, with sharp talons and strong wings. They can catch prey in mid-air, and some species can even catch fish right out o' the water!\n", - "5. **Monogamous**: Many owl species be monogamous, which means they mate fer life. Some species even stay together fer many years, which be a rare thing in the animal kingdom.\n", + "* Barn Owl: A medium-sized owl with a heart-shaped face and a screechin' call.\n", + "* Tawny Owl: A large owl with a distinctive hootin' call and a reddish-brown plumage.\n", + "* Great Horned Owl: A big owl with ear tufts and a deep hootin' call.\n", + "* Snowy Owl: A white owl with a round face and a soft, hootin' call.\n", "\n", - "So there ye have it, matey! Owls be amazin' creatures, and there be much more to learn about 'em. Now go forth and spread the word about these fascinatin' birds!" + "Owls be found all over the world, in a variety o' habitats, from forests to deserts. They be an important part o' many ecosystems, helpin' to keep populations o' small mammals and birds under control.\n", + "\n", + "So there ye have it, matey! Owls be amazin' creatures, with their big eyes, silent wings, and sharp talons. Now go forth and spread the word about these fascinatin' birds!" ] } ], @@ -289,7 +295,7 @@ { "data": { "text/plain": [ - "AIMessage(content='The capital of France is Paris.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 1, 'completion_tokens': 7, 'completion_tokens_after_first_per_sec': 326.8246386410566, 'completion_tokens_after_first_per_sec_first_ten': 0, 'completion_tokens_per_sec': 28.56260299906024, 'end_time': 1731001010.9238367, 'is_last_response': True, 'prompt_tokens': 42, 'start_time': 1731001010.678761, 'time_to_first_token': 0.22671723365783691, 'total_latency': 0.24507570266723633, 'total_tokens': 49, 'total_tokens_per_sec': 199.93822099342168}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731001010}, id='c17f7e9b-2db4-4c53-bf1a-980c773cf554')" + "AIMessage(content='The capital of France is Paris.', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 1, 'completion_tokens': 7, 'completion_tokens_after_first_per_sec': 442.126212227688, 'completion_tokens_after_first_per_sec_first_ten': 0, 'completion_tokens_per_sec': 46.28540439646366, 'end_time': 1731535343.0321083, 'is_last_response': True, 'prompt_tokens': 42, 'start_time': 1731535342.8808727, 'time_to_first_token': 0.137664794921875, 'total_latency': 0.15123558044433594, 'total_tokens': 49, 'total_tokens_per_sec': 323.99783077524563}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731535342}, id='c4b8c714-df38-4206-9aa8-fc8231f7275a')" ] }, "execution_count": 7, @@ -350,20 +356,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Tool calling\n", - "\n", - "SambaNovaCloud supports tool calling" + "## Tool calling" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "from datetime import datetime\n", "\n", - "from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage\n", + "from langchain_core.messages import HumanMessage, ToolMessage\n", "from langchain_core.tools import tool\n", "\n", "\n", @@ -371,7 +375,9 @@ "def get_time(kind: str = \"both\") -> str:\n", " \"\"\"Returns current date, current time or both.\n", " Args:\n", - " kind: date, time or both\n", + " kind(str): date, time or both\n", + " Returns:\n", + " str: current date, current time or both\n", " \"\"\"\n", " if kind == \"date\":\n", " date = datetime.now().strftime(\"%m/%d/%Y\")\n", @@ -400,7 +406,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -414,16 +420,16 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Intermediate model response: [{'name': 'get_time', 'args': {'kind': 'date'}, 'id': 'call_4092d5dd21cd4eb494', 'type': 'tool_call'}]\n", - "Tool output: Current date: 11/07/2024\n", - "final response: The meeting will be exactly two weeks from today, which would be 25/07/2024.\n" + "Intermediate model response: [{'name': 'get_time', 'args': {'kind': 'date'}, 'id': 'call_7352ce7a18e24a7c9d', 'type': 'tool_call'}]\n", + "Tool output: Current date: 11/13/2024\n", + "final response: The meeting should be scheduled for two weeks from November 13th, 2024.\n" ] } ], @@ -433,11 +439,108 @@ " print(f\"Intermediate model response: {response.tool_calls}\")\n", " messages.append(response)\n", " messages = invoke_tools(response.tool_calls, messages)\n", - "response = llm_with_tools.invoke(messages)\n", + " response = llm_with_tools.invoke(messages)\n", "\n", "print(f\"final response: {response.content}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Joke(setup='Why did the cat join a band?', punchline='Because it wanted to be the purr-cussionist!')" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "class Joke(BaseModel):\n", + " \"\"\"Joke to tell user.\"\"\"\n", + "\n", + " setup: str = Field(description=\"The setup of the joke\")\n", + " punchline: str = Field(description=\"The punchline to the joke\")\n", + "\n", + "\n", + "structured_llm = llm.with_structured_output(Joke)\n", + "\n", + "structured_llm.invoke(\"Tell me a joke about cats\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Input Image" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [], + "source": [ + "multimodal_llm = ChatSambaNovaCloud(\n", + " model=\"Llama-3.2-11B-Vision-Instruct\",\n", + " max_tokens=1024,\n", + " temperature=0.7,\n", + " top_k=1,\n", + " top_p=0.01,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The weather in this image is a serene and peaceful atmosphere, with a blue sky and white clouds, suggesting a pleasant day with mild temperatures and gentle breezes.\n" + ] + } + ], + "source": [ + "import base64\n", + "\n", + "import httpx\n", + "\n", + "image_url = (\n", + " \"https://images.pexels.com/photos/147411/italy-mountains-dawn-daybreak-147411.jpeg\"\n", + ")\n", + "image_data = base64.b64encode(httpx.get(image_url).content).decode(\"utf-8\")\n", + "\n", + "message = HumanMessage(\n", + " content=[\n", + " {\"type\": \"text\", \"text\": \"describe the weather in this image in 1 sentence\"},\n", + " {\n", + " \"type\": \"image_url\",\n", + " \"image_url\": {\"url\": f\"data:image/jpeg;base64,{image_data}\"},\n", + " },\n", + " ],\n", + ")\n", + "response = multimodal_llm.invoke([message])\n", + "print(response.content)" + ] + }, { "cell_type": "markdown", "metadata": {}, From 065334ef166fb205bae2e57cc95fc32b094febad Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:12:28 -0500 Subject: [PATCH 19/30] updated docs links --- docs/docs/integrations/chat/sambanova.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index f81a6c2d02b05..cca46f920eccc 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -19,7 +19,7 @@ "source": [ "# ChatSambaNovaCloud\n", "\n", - "This will help you getting started with SambaNovaCloud [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatSambaNovaCloud features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html).\n", + "This will help you getting started with SambaNovaCloud [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatSambaNovaCloud features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html).\n", "\n", "**[SambaNova](https://sambanova.ai/)'s** [SambaNova Cloud](https://cloud.sambanova.ai/) is a platform for performing inference with open-source models\n", "\n", @@ -28,7 +28,7 @@ "\n", "| Class | Package | Local | Serializable | JS support | Package downloads | Package latest |\n", "| :--- | :--- | :---: | :---: | :---: | :---: | :---: |\n", - "| [ChatSambaNovaCloud](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html) | [langchain-community](https://python.langchain.com/v0.2/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", + "| [ChatSambaNovaCloud](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", "\n", "### Model features\n", "\n", @@ -547,7 +547,7 @@ "source": [ "## API reference\n", "\n", - "For detailed documentation of all ChatSambaNovaCloud features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html" + "For detailed documentation of all ChatSambaNovaCloud features and configurations head to the API reference: https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaNovaCloud.html" ] } ], From 7b146345468288a603ed63fb34bb2c8399db5151 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:52:07 -0500 Subject: [PATCH 20/30] fmt docstrings --- .../chat_models/sambanova.py | 58 ++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index bfceb052ac8a9..bff480c865c44 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -166,8 +166,10 @@ class ChatSambaNovaCloud(BaseChatModel): top_k = model top k, stream_options = include usage to get generation metrics ) + Invoke: .. code-block:: python + messages = [ SystemMessage(content="your are an AI assistant."), HumanMessage(content="tell me a joke."), @@ -177,41 +179,41 @@ class ChatSambaNovaCloud(BaseChatModel): Stream: .. code-block:: python - for chunk in chat.stream(messages): - print(chunk.content, end="", flush=True) + for chunk in chat.stream(messages): + print(chunk.content, end="", flush=True) Async: .. code-block:: python - response = chat.ainvoke(messages) - await response + response = chat.ainvoke(messages) + await response Tool calling: .. code-block:: python - from pydantic import BaseModel, Field + from pydantic import BaseModel, Field - class GetWeather(BaseModel): - '''Get the current weather in a given location''' + class GetWeather(BaseModel): + '''Get the current weather in a given location''' - location: str = Field( - ..., - description="The city and state, e.g. Los Angeles, CA" - ) + location: str = Field( + ..., + description="The city and state, e.g. Los Angeles, CA" + ) - llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) - ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") - ai_msg.tool_calls + llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) + ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") + ai_msg.tool_calls - .. code-block:: python + .. code-block:: none - [ - { - 'name': 'GetWeather', - 'args': {'location': 'Los Angeles, CA'}, - 'id': 'call_adf61180ea2b4d228a' - } - ] + [ + { + 'name': 'GetWeather', + 'args': {'location': 'Los Angeles, CA'}, + 'id': 'call_adf61180ea2b4d228a' + } + ] Structured output: .. code-block:: python @@ -238,15 +240,17 @@ class Joke(BaseModel): Token usage: .. code-block:: python - response = chat.invoke(messages) - print(response.response_metadata["usage"]["prompt_tokens"] - print(response.response_metadata["usage"]["total_tokens"] + + response = chat.invoke(messages) + print(response.response_metadata["usage"]["prompt_tokens"] + print(response.response_metadata["usage"]["total_tokens"] Response metadata .. code-block:: python - response = chat.invoke(messages) - print(response.response_metadata) + response = chat.invoke(messages) + print(response.response_metadata) + """ sambanova_url: str = Field(default="") From ba4c22c16298ab85355b9e29a08c7d8574ef3254 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:54:21 -0500 Subject: [PATCH 21/30] fmt docstrings --- libs/community/langchain_community/chat_models/sambanova.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index bff480c865c44..3416fb5ee463d 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -573,6 +573,7 @@ class AnswerWithJustification(BaseModel): Example: schema=None, method="json_mode", include_raw=True: .. code-block:: + from langchain_community.chat_models import ChatSambaNovaCloud llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) @@ -594,6 +595,7 @@ class AnswerWithJustification(BaseModel): Example: schema=None, method="json_schema", include_raw=True: .. code-block:: + from langchain_community.chat_models import ChatSambaNovaCloud class AnswerWithJustification(BaseModel): From 55734ab1c9f5099ad90dd5beb9c8e960280b6202 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 13 Nov 2024 17:54:38 -0500 Subject: [PATCH 22/30] fmt docstrings --- libs/community/langchain_community/chat_models/sambanova.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 3416fb5ee463d..e63340f3b047f 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -573,7 +573,7 @@ class AnswerWithJustification(BaseModel): Example: schema=None, method="json_mode", include_raw=True: .. code-block:: - + from langchain_community.chat_models import ChatSambaNovaCloud llm = ChatSambaNovaCloud(model="Meta-Llama-3.1-70B-Instruct", temperature=0) @@ -595,7 +595,7 @@ class AnswerWithJustification(BaseModel): Example: schema=None, method="json_schema", include_raw=True: .. code-block:: - + from langchain_community.chat_models import ChatSambaNovaCloud class AnswerWithJustification(BaseModel): From 224710aa7973e3a204a75d77d6829d3dd2383d77 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 14 Nov 2024 12:19:47 -0500 Subject: [PATCH 23/30] add structured output sambastudio chat model --- .../chat_models/sambanova.py | 396 ++++++++++++++++-- 1 file changed, 369 insertions(+), 27 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index a55ff1f7d12af..93b7f919be80b 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -615,6 +615,7 @@ class AnswerWithJustification(BaseModel): # 'parsed': AnswerWithJustification(answer='They are the same weight', justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'), # 'parsing_error': None # } + """ # noqa: E501 if kwargs: raise ValueError(f"Received unsupported arguments {kwargs}") @@ -962,7 +963,9 @@ class ChatSambaStudio(BaseChatModel): ``SAMBASTUDIO_API_KEY`` set with your SambaStudio deployed endpoint Key. https://docs.sambanova.ai/sambastudio/latest/index.html Example: + .. code-block:: python + ChatSambaStudio( sambastudio_url = set with your SambaStudio deployed endpoint URL, sambastudio_api_key = set with your SambaStudio deployed endpoint Key. @@ -1036,8 +1039,10 @@ class ChatSambaStudio(BaseChatModel): set to false or for StandAlone v1 and v2 endpoints) model_kwargs: Optional = Extra Key word arguments to pass to the model. ) + Invoke: .. code-block:: python + messages = [ SystemMessage(content="your are an AI assistant."), HumanMessage(content="tell me a joke."), @@ -1047,53 +1052,77 @@ class ChatSambaStudio(BaseChatModel): Stream: .. code-block:: python - for chunk in chat.stream(messages): - print(chunk.content, end="", flush=True) + for chunk in chat.stream(messages): + print(chunk.content, end="", flush=True) Async: .. code-block:: python - response = chat.ainvoke(messages) - await response + response = chat.ainvoke(messages) + await response Tool calling: .. code-block:: python - from pydantic import BaseModel, Field + from pydantic import BaseModel, Field - class GetWeather(BaseModel): - '''Get the current weather in a given location''' + class GetWeather(BaseModel): + '''Get the current weather in a given location''' - location: str = Field( - ..., - description="The city and state, e.g. Los Angeles, CA" - ) + location: str = Field( + ..., + description="The city and state, e.g. Los Angeles, CA" + ) - llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) - ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") - ai_msg.tool_calls + llm_with_tools = llm.bind_tools([GetWeather, GetPopulation]) + ai_msg = llm_with_tools.invoke("Should I bring my umbrella today in LA?") + ai_msg.tool_calls .. code-block:: python - [ - { - 'name': 'GetWeather', - 'args': {'location': 'Los Angeles, CA'}, - 'id': 'call_adf61180ea2b4d228a' - } - ] + [ + { + 'name': 'GetWeather', + 'args': {'location': 'Los Angeles, CA'}, + 'id': 'call_adf61180ea2b4d228a' + } + ] + + Structured output: + .. code-block:: python + + from typing import Optional + + from pydantic import BaseModel, Field + + class Joke(BaseModel): + '''Joke to tell user.''' + + setup: str = Field(description="The setup of the joke") + punchline: str = Field(description="The punchline to the joke") + + structured_model = llm.with_structured_output(Joke) + structured_model.invoke("Tell me a joke about cats") + + .. code-block:: python + + Joke(setup="Why did the cat join a band?", + punchline="Because it wanted to be the purr-cussionist!") + + See ``ChatSambaStudio.with_structured_output()`` for more. Token usage: .. code-block:: python - response = chat.invoke(messages) - print(response.response_metadata["usage"]["prompt_tokens"] - print(response.response_metadata["usage"]["total_tokens"] + + response = chat.invoke(messages) + print(response.response_metadata["usage"]["prompt_tokens"] + print(response.response_metadata["usage"]["total_tokens"] Response metadata .. code-block:: python - response = chat.invoke(messages) - print(response.response_metadata) + response = chat.invoke(messages) + print(response.response_metadata) """ sambastudio_url: str = Field(default="") @@ -1248,6 +1277,319 @@ def bind_tools( kwargs["parallel_tool_calls"] = parallel_tool_calls return super().bind(tools=formatted_tools, **kwargs) + def with_structured_output( + self, + schema: Optional[Union[Dict[str, Any], Type[BaseModel]]] = None, + *, + method: Literal[ + "function_calling", "json_mode", "json_schema" + ] = "function_calling", + include_raw: bool = False, + **kwargs: Any, + ) -> Runnable[LanguageModelInput, Union[Dict[str, Any], BaseModel]]: + """Model wrapper that returns outputs formatted to match the given schema. + + Args: + schema: + The output schema. Can be passed in as: + - an OpenAI function/tool schema, + - a JSON Schema, + - a TypedDict class, + - or a Pydantic class. + If ``schema`` is a Pydantic class then the model output will be a + Pydantic instance of that class, and the model-generated fields will be + validated by the Pydantic class. Otherwise the model output will be a + dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` + for more on how to properly specify types and descriptions of + schema fields when specifying a Pydantic or TypedDict class. + + method: + The method for steering model generation, either "function_calling" + "json_mode" or "json_schema". + If "function_calling" then the schema will be converted + to an OpenAI function and the returned model will make use of the + function-calling API. If "json_mode" or "json_schema" then OpenAI's + JSON mode will be used. + Note that if using "json_mode" or "json_schema" then you must include instructions + for formatting the output into the desired schema into the model call. + + include_raw: + If False then only the parsed structured output is returned. If + an error occurs during model output parsing it will be raised. If True + then both the raw model response (a BaseMessage) and the parsed model + response will be returned. If an error occurs during output parsing it + will be caught and returned as well. The final output is always a dict + with keys "raw", "parsed", and "parsing_error". + + Returns: + A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. + + If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs + an instance of ``schema`` (i.e., a Pydantic object). + + Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + + If ``include_raw`` is True, then Runnable outputs a dict with keys: + - ``"raw"``: BaseMessage + - ``"parsed"``: None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. + - ``"parsing_error"``: Optional[BaseException] + + Example: schema=Pydantic class, method="function_calling", include_raw=False: + .. code-block:: python + + from typing import Optional + + from langchain_community.chat_models import ChatSambaStudio + from pydantic import BaseModel, Field + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str = Field( + description="A justification for the answer." + ) + + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + + # -> AnswerWithJustification( + # answer='They weigh the same', + # justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same.' + # ) + + Example: schema=Pydantic class, method="function_calling", include_raw=True: + .. code-block:: python + + from langchain_community.chat_models import ChatSambaStudio + from pydantic import BaseModel + + + class AnswerWithJustification(BaseModel): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: str + + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, include_raw=True + ) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'raw': AIMessage(content='', additional_kwargs={'tool_calls': [{'function': {'arguments': '{"answer": "They weigh the same.", "justification": "A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount."}', 'name': 'AnswerWithJustification'}, 'id': 'call_17a431fc6a4240e1bd', 'type': 'function'}]}, response_metadata={'finish_reason': 'tool_calls', 'usage': {'acceptance_rate': 5, 'completion_tokens': 53, 'completion_tokens_after_first_per_sec': 343.7964936837758, 'completion_tokens_after_first_per_sec_first_ten': 439.1205661878638, 'completion_tokens_per_sec': 162.8511306784833, 'end_time': 1731527851.0698032, 'is_last_response': True, 'prompt_tokens': 213, 'start_time': 1731527850.7137961, 'time_to_first_token': 0.20475482940673828, 'total_latency': 0.32545061111450196, 'total_tokens': 266, 'total_tokens_per_sec': 817.3283162354066}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731527850}, id='95667eaf-447f-4b53-bb6e-b6e1094ded88', tool_calls=[{'name': 'AnswerWithJustification', 'args': {'answer': 'They weigh the same.', 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.'}, 'id': 'call_17a431fc6a4240e1bd', 'type': 'tool_call'}]), + # 'parsed': AnswerWithJustification(answer='They weigh the same.', justification='A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.'), + # 'parsing_error': None + # } + + Example: schema=TypedDict class, method="function_calling", include_raw=False: + .. code-block:: python + + # IMPORTANT: If you are using Python <=3.8, you need to import Annotated + # from typing_extensions, not from typing. + from typing_extensions import Annotated, TypedDict + + from langchain_community.chat_models import ChatSambaStudio + + + class AnswerWithJustification(TypedDict): + '''An answer to the user question along with justification for the answer.''' + + answer: str + justification: Annotated[ + Optional[str], None, "A justification for the answer." + ] + + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.' + # } + + Example: schema=OpenAI function schema, method="function_calling", include_raw=False: + .. code-block:: python + + from langchain_community.chat_models import ChatSambaStudio + + oai_schema = { + 'name': 'AnswerWithJustification', + 'description': 'An answer to the user question along with justification for the answer.', + 'parameters': { + 'type': 'object', + 'properties': { + 'answer': {'type': 'string'}, + 'justification': {'description': 'A justification for the answer.', 'type': 'string'} + }, + 'required': ['answer'] + } + } + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(oai_schema) + + structured_llm.invoke( + "What weighs more a pound of bricks or a pound of feathers" + ) + # -> { + # 'answer': 'They weigh the same', + # 'justification': 'A pound is a unit of weight or mass, so one pound of bricks and one pound of feathers both weigh the same amount.' + # } + + Example: schema=Pydantic class, method="json_mode", include_raw=True: + .. code-block:: + + from langchain_community.chat_models import ChatSambaStudio + from pydantic import BaseModel + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output( + AnswerWithJustification, + method="json_mode", + include_raw=True + ) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 5.3125, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 292.65701089829776, 'completion_tokens_after_first_per_sec_first_ten': 346.43324678555325, 'completion_tokens_per_sec': 200.012158915008, 'end_time': 1731528071.1708555, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528070.737394, 'time_to_first_token': 0.16693782806396484, 'total_latency': 0.3949759876026827, 'total_tokens': 149, 'total_tokens_per_sec': 377.2381225105847}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528070}, id='83208297-3eb9-4021-a856-ca78a15758df'), + # 'parsed': AnswerWithJustification(answer='They are the same weight', justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'), + # 'parsing_error': None + # } + + Example: schema=None, method="json_mode", include_raw=True: + .. code-block:: + + from langchain_community.chat_models import ChatSambaStudio + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(method="json_mode", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 4.722222222222222, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 357.1315485254867, 'completion_tokens_after_first_per_sec_first_ten': 416.83279609305305, 'completion_tokens_per_sec': 240.92819585198137, 'end_time': 1731528164.8474727, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528164.4906917, 'time_to_first_token': 0.13837409019470215, 'total_latency': 0.3278985247892492, 'total_tokens': 149, 'total_tokens_per_sec': 454.4088757208256}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528164}, id='15261eaf-8a25-42ef-8ed5-f63d8bf5b1b0'), + # 'parsed': { + # 'answer': 'They are the same weight', + # 'justification': 'A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'}, + # }, + # 'parsing_error': None + # } + + Example: schema=None, method="json_schema", include_raw=True: + .. code-block:: + + from langchain_community.chat_models import ChatSambaStudio + + class AnswerWithJustification(BaseModel): + answer: str + justification: str + + llm = ChatSambaStudio(model="Meta-Llama-3.1-70B-Instruct", temperature=0) + structured_llm = llm.with_structured_output(AnswerWithJustification, method="json_schema", include_raw=True) + + structured_llm.invoke( + "Answer the following question. " + "Make sure to return a JSON blob with keys 'answer' and 'justification'.\n\n" + "What's heavier a pound of bricks or a pound of feathers?" + ) + # -> { + # 'raw': AIMessage(content='{\n "answer": "They are the same weight",\n "justification": "A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities."\n}', additional_kwargs={}, response_metadata={'finish_reason': 'stop', 'usage': {'acceptance_rate': 5.3125, 'completion_tokens': 79, 'completion_tokens_after_first_per_sec': 292.65701089829776, 'completion_tokens_after_first_per_sec_first_ten': 346.43324678555325, 'completion_tokens_per_sec': 200.012158915008, 'end_time': 1731528071.1708555, 'is_last_response': True, 'prompt_tokens': 70, 'start_time': 1731528070.737394, 'time_to_first_token': 0.16693782806396484, 'total_latency': 0.3949759876026827, 'total_tokens': 149, 'total_tokens_per_sec': 377.2381225105847}, 'model_name': 'Meta-Llama-3.1-70B-Instruct', 'system_fingerprint': 'fastcoe', 'created': 1731528070}, id='83208297-3eb9-4021-a856-ca78a15758df'), + # 'parsed': AnswerWithJustification(answer='They are the same weight', justification='A pound is a unit of weight or mass, so a pound of bricks and a pound of feathers both weigh the same amount, one pound. The difference is in their density and volume. A pound of feathers would take up more space than a pound of bricks due to the difference in their densities.'), + # 'parsing_error': None + # } + + """ # noqa: E501 + if kwargs: + raise ValueError(f"Received unsupported arguments {kwargs}") + is_pydantic_schema = _is_pydantic_class(schema) + if method == "function_calling": + if schema is None: + raise ValueError( + "schema must be specified when method is 'function_calling'. " + "Received None." + ) + tool_name = convert_to_openai_tool(schema)["function"]["name"] + llm = self.bind_tools([schema], tool_choice=tool_name) + if is_pydantic_schema: + output_parser: OutputParserLike[Any] = PydanticToolsParser( + tools=[schema], # type: ignore[list-item] + first_tool_only=True, + ) + else: + output_parser = JsonOutputKeyToolsParser( + key_name=tool_name, first_tool_only=True + ) + elif method == "json_mode": + llm = self + # TODO bind response format when json mode available by API + # llm = self.bind(response_format={"type": "json_object"}) + if is_pydantic_schema: + schema = cast(Type[BaseModel], schema) + output_parser = PydanticOutputParser(pydantic_object=schema) + else: + output_parser = JsonOutputParser() + + elif method == "json_schema": + if schema is None: + raise ValueError( + "schema must be specified when method is not 'json_mode'. " + "Received None." + ) + llm = self + # TODO bind response format when json schema available by API, + # update example + # llm = self.bind( + # response_format={"type": "json_object", "json_schema": schema} + # ) + if is_pydantic_schema: + schema = cast(Type[BaseModel], schema) + output_parser = PydanticOutputParser(pydantic_object=schema) + else: + output_parser = JsonOutputParser() + else: + raise ValueError( + f"Unrecognized method argument. Expected one of 'function_calling' or " + f"'json_mode'. Received: '{method}'" + ) + + if include_raw: + parser_assign = RunnablePassthrough.assign( + parsed=itemgetter("raw") | output_parser, parsing_error=lambda _: None + ) + parser_none = RunnablePassthrough.assign(parsed=lambda _: None) + parser_with_fallback = parser_assign.with_fallbacks( + [parser_none], exception_key="parsing_error" + ) + return RunnableMap(raw=llm) | parser_with_fallback + else: + return llm | output_parser + def _get_role(self, message: BaseMessage) -> str: """ Get the role of LangChain BaseMessage @@ -1298,8 +1640,8 @@ def _messages_to_string(self, messages: List[BaseMessage]) -> str: "role": self._get_role(message), "content": message.content, } + # TODO add tools msgs id and assistant msgs tool calls ) - # TODO add tools msgs id and assistant msgs tool calls messages_string = json.dumps(messages_dict) else: messages_string = self.special_tokens["start"] From b346fb0a75922bd1924edc242c48c29a3d26cb09 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Thu, 14 Nov 2024 12:33:43 -0500 Subject: [PATCH 24/30] add structured output sambastudio chat model docs notebook --- docs/docs/integrations/chat/sambastudio.ipynb | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/docs/docs/integrations/chat/sambastudio.ipynb b/docs/docs/integrations/chat/sambastudio.ipynb index dcf1c450c55c6..5527baa6417d8 100644 --- a/docs/docs/integrations/chat/sambastudio.ipynb +++ b/docs/docs/integrations/chat/sambastudio.ipynb @@ -19,7 +19,7 @@ "source": [ "# ChatSambaStudio\n", "\n", - "This will help you getting started with SambaStudio [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatStudio features and configurations head to the [API reference](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html).\n", + "This will help you getting started with SambaStudio [chat models](/docs/concepts/chat_models). For detailed documentation of all ChatStudio features and configurations head to the [API reference](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html).\n", "\n", "**[SambaNova](https://sambanova.ai/)'s** [SambaStudio](https://docs.sambanova.ai/sambastudio/latest/sambastudio-intro.html) SambaStudio is a rich, GUI-based platform that provides the functionality to train, deploy, and manage models in SambaNova [DataScale](https://sambanova.ai/products/datascale) systems.\n", "\n", @@ -28,13 +28,13 @@ "\n", "| Class | Package | Local | Serializable | JS support | Package downloads | Package latest |\n", "| :--- | :--- | :---: | :---: | :---: | :---: | :---: |\n", - "| [ChatSambaStudio](https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", + "| [ChatSambaStudio](https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html) | [langchain-community](https://python.langchain.com/api_reference/community/index.html) | ❌ | ❌ | ❌ | ![PyPI - Downloads](https://img.shields.io/pypi/dm/langchain_community?style=flat-square&label=%20) | ![PyPI - Version](https://img.shields.io/pypi/v/langchain_community?style=flat-square&label=%20) |\n", "\n", "### Model features\n", "\n", "| [Tool calling](/docs/how_to/tool_calling) | [Structured output](/docs/how_to/structured_output/) | JSON mode | [Image input](/docs/how_to/multimodal_inputs/) | Audio input | Video input | [Token-level streaming](/docs/how_to/chat_streaming/) | Native async | [Token usage](/docs/how_to/chat_token_usage_tracking/) | [Logprobs](/docs/how_to/logprobs/) |\n", "| :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: |\n", - "| ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", + "| ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | \n", "\n", "## Setup\n", "\n", @@ -439,13 +439,51 @@ "print(f\"final response: {response.content}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Structured Outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Joke(setup='Why did the cat join a band?', punchline='Because it wanted to be the purr-cussionist!')" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from pydantic import BaseModel, Field\n", + "\n", + "\n", + "class Joke(BaseModel):\n", + " \"\"\"Joke to tell user.\"\"\"\n", + "\n", + " setup: str = Field(description=\"The setup of the joke\")\n", + " punchline: str = Field(description=\"The punchline to the joke\")\n", + "\n", + "\n", + "structured_llm = llm.with_structured_output(Joke)\n", + "\n", + "structured_llm.invoke(\"Tell me a joke about cats\")" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "## API reference\n", "\n", - "For detailed documentation of all ChatSambaStudio features and configurations head to the API reference: https://api.python.langchain.com/en/latest/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html" + "For detailed documentation of all ChatSambaStudio features and configurations head to the API reference: https://python.langchain.com/api_reference/community/chat_models/langchain_community.chat_models.sambanova.ChatSambaStudio.html" ] } ], From 28e1a7e45873415c01cfb5b7ca733bfce750d210 Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Fri, 15 Nov 2024 15:28:00 -0500 Subject: [PATCH 25/30] fix typo --- docs/docs/integrations/chat/sambanova.ipynb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/docs/integrations/chat/sambanova.ipynb b/docs/docs/integrations/chat/sambanova.ipynb index cca46f920eccc..ecfab9b25b4a2 100644 --- a/docs/docs/integrations/chat/sambanova.ipynb +++ b/docs/docs/integrations/chat/sambanova.ipynb @@ -116,12 +116,11 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "# from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", - "from langchain_chat_models import ChatSambaNovaCloud\n", + "from langchain_community.chat_models.sambanova import ChatSambaNovaCloud\n", "\n", "llm = ChatSambaNovaCloud(\n", " model=\"Meta-Llama-3.1-70B-Instruct\",\n", From e55c580a54bdc842deaea6efd2ed7be2cc4a76cf Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Mon, 18 Nov 2024 11:04:30 -0500 Subject: [PATCH 26/30] fmt --- .../chat_models/sambanova.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index e63340f3b047f..358650a892e9b 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -113,8 +113,8 @@ class ChatSambaNovaCloud(BaseChatModel): Setup: To use, you should have the environment variables: - ``SAMBANOVA_URL`` set with your SambaNova Cloud URL. - ``SAMBANOVA_API_KEY`` set with your SambaNova Cloud API Key. + `SAMBANOVA_URL` set with your SambaNova Cloud URL. + `SAMBANOVA_API_KEY` set with your SambaNova Cloud API Key. http://cloud.sambanova.ai/ Example: .. code-block:: python @@ -236,7 +236,7 @@ class Joke(BaseModel): Joke(setup="Why did the cat join a band?", punchline="Because it wanted to be the purr-cussionist!") - See ``ChatSambanovaCloud.with_structured_output()`` for more. + See `ChatSambanovaCloud.with_structured_output()` for more. Token usage: .. code-block:: python @@ -387,8 +387,8 @@ def with_structured_output( - an OpenAI function/tool schema, - a JSON Schema, - a TypedDict class, - - or a Pydantic class. - If ``schema`` is a Pydantic class then the model output will be a + - or a Pydantic.BaseModel class. + If `schema` is a Pydantic class then the model output will be a Pydantic instance of that class, and the model-generated fields will be validated by the Pydantic class. Otherwise the model output will be a dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` @@ -416,15 +416,15 @@ def with_structured_output( Returns: A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. - If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs - an instance of ``schema`` (i.e., a Pydantic object). + If `include_raw` is False and `schema` is a Pydantic class, Runnable outputs + an instance of `schema` (i.e., a Pydantic object). - Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + Otherwise, if `include_raw` is False then Runnable outputs a dict. - If ``include_raw`` is True, then Runnable outputs a dict with keys: - - ``"raw"``: BaseMessage - - ``"parsed"``: None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. - - ``"parsing_error"``: Optional[BaseException] + If `include_raw` is True, then Runnable outputs a dict with keys: + - `"raw"`: BaseMessage + - `"parsed"`: None if there was a parsing error, otherwise the type depends on the `schema` as described above. + - `"parsing_error"`: Optional[BaseException] Example: schema=Pydantic class, method="function_calling", include_raw=False: .. code-block:: python @@ -616,20 +616,20 @@ class AnswerWithJustification(BaseModel): # 'parsing_error': None # } """ # noqa: E501 - if kwargs: + if kwargs is not None: raise ValueError(f"Received unsupported arguments {kwargs}") is_pydantic_schema = _is_pydantic_class(schema) if method == "function_calling": if schema is None: raise ValueError( - "schema must be specified when method is 'function_calling'. " + "`schema` must be specified when method is `function_calling`. " "Received None." ) tool_name = convert_to_openai_tool(schema)["function"]["name"] llm = self.bind_tools([schema], tool_choice=tool_name) if is_pydantic_schema: output_parser: OutputParserLike[Any] = PydanticToolsParser( - tools=[schema], # type: ignore[list-item] + tools=[schema], first_tool_only=True, ) else: @@ -649,7 +649,7 @@ class AnswerWithJustification(BaseModel): elif method == "json_schema": if schema is None: raise ValueError( - "schema must be specified when method is not 'json_mode'. " + "`schema` must be specified when method is not `json_mode`. " "Received None." ) llm = self @@ -665,8 +665,8 @@ class AnswerWithJustification(BaseModel): output_parser = JsonOutputParser() else: raise ValueError( - f"Unrecognized method argument. Expected one of 'function_calling' or " - f"'json_mode'. Received: '{method}'" + f"Unrecognized method argument. Expected one of `function_calling` or " + f"`json_mode`. Received: `{method}`" ) if include_raw: From 8781a4c67bf29924c1501b12e929220afcf418be Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Tue, 19 Nov 2024 16:29:12 -0500 Subject: [PATCH 27/30] tool callig samabstudio updated --- .../chat_models/sambanova.py | 78 +++++++++++++------ 1 file changed, 53 insertions(+), 25 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 89e34ab2dc7d5..99b9a431f52a3 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -623,8 +623,7 @@ class AnswerWithJustification(BaseModel): if method == "function_calling": if schema is None: raise ValueError( - "`schema` must be specified when method is `function_calling`. " - "Received None." + "`schema` must be specified when method is `function_calling`. Received None." ) tool_name = convert_to_openai_tool(schema)["function"]["name"] llm = self.bind_tools([schema], tool_choice=tool_name) @@ -650,8 +649,7 @@ class AnswerWithJustification(BaseModel): elif method == "json_schema": if schema is None: raise ValueError( - "`schema` must be specified when method is not `json_mode`. " - "Received None." + "`schema` must be specified when method is not `json_mode`. Received None." ) llm = self # TODO bind response format when json schema available by API, @@ -959,8 +957,8 @@ class ChatSambaStudio(BaseChatModel): Setup: To use, you should have the environment variables: - ``SAMBASTUDIO_URL`` set with your SambaStudio deployed endpoint URL. - ``SAMBASTUDIO_API_KEY`` set with your SambaStudio deployed endpoint Key. + `SAMBASTUDIO_URL` set with your SambaStudio deployed endpoint URL. + `SAMBASTUDIO_API_KEY` set with your SambaStudio deployed endpoint Key. https://docs.sambanova.ai/sambastudio/latest/index.html Example: @@ -1109,7 +1107,7 @@ class Joke(BaseModel): Joke(setup="Why did the cat join a band?", punchline="Because it wanted to be the purr-cussionist!") - See ``ChatSambaStudio.with_structured_output()`` for more. + See `ChatSambaStudio.with_structured_output()` for more. Token usage: .. code-block:: python @@ -1296,7 +1294,7 @@ def with_structured_output( - a JSON Schema, - a TypedDict class, - or a Pydantic class. - If ``schema`` is a Pydantic class then the model output will be a + If `schema` is a Pydantic class then the model output will be a Pydantic instance of that class, and the model-generated fields will be validated by the Pydantic class. Otherwise the model output will be a dict and will not be validated. See :meth:`langchain_core.utils.function_calling.convert_to_openai_tool` @@ -1324,15 +1322,15 @@ def with_structured_output( Returns: A Runnable that takes same inputs as a :class:`langchain_core.language_models.chat.BaseChatModel`. - If ``include_raw`` is False and ``schema`` is a Pydantic class, Runnable outputs - an instance of ``schema`` (i.e., a Pydantic object). + If `include_raw` is False and `schema` is a Pydantic class, Runnable outputs + an instance of `schema` (i.e., a Pydantic object). - Otherwise, if ``include_raw`` is False then Runnable outputs a dict. + Otherwise, if `include_raw` is False then Runnable outputs a dict. - If ``include_raw`` is True, then Runnable outputs a dict with keys: - - ``"raw"``: BaseMessage - - ``"parsed"``: None if there was a parsing error, otherwise the type depends on the ``schema`` as described above. - - ``"parsing_error"``: Optional[BaseException] + If `include_raw` is True, then Runnable outputs a dict with keys: + - `"raw"`: BaseMessage + - `"parsed"`: None if there was a parsing error, otherwise the type depends on the `schema` as described above. + - `"parsing_error"`: Optional[BaseException] Example: schema=Pydantic class, method="function_calling", include_raw=False: .. code-block:: python @@ -1600,9 +1598,7 @@ def _get_role(self, message: BaseMessage) -> str: Returns: str: Role of the LangChain BaseMessage """ - if isinstance(message, ChatMessage): - role = message.role - elif isinstance(message, SystemMessage): + if isinstance(message, SystemMessage): role = "system" elif isinstance(message, HumanMessage): role = "user" @@ -1610,11 +1606,13 @@ def _get_role(self, message: BaseMessage) -> str: role = "assistant" elif isinstance(message, ToolMessage): role = "tool" + elif isinstance(message, ChatMessage): + role = message.role else: raise TypeError(f"Got unknown type {message}") return role - def _messages_to_string(self, messages: List[BaseMessage]) -> str: + def _messages_to_string(self, messages: List[BaseMessage], **kwargs: Any) -> str: """ Convert a list of BaseMessages to a: - dumped json string with Role / content dict structure @@ -1632,18 +1630,47 @@ def _messages_to_string(self, messages: List[BaseMessage]) -> str: messages_dict: Dict[str, Any] = { "conversation_id": "sambaverse-conversation-id", "messages": [], + **kwargs, } for message in messages: - messages_dict["messages"].append( - { + if isinstance(message, AIMessage): + message_dict = { "message_id": message.id, "role": self._get_role(message), "content": message.content, } - # TODO add tools msgs id and assistant msgs tool calls - ) + if "tool_calls" in message.additional_kwargs: + message_dict["tool_calls"] = message.additional_kwargs[ + "tool_calls" + ] + if message_dict["content"] == "": + message_dict["content"] = None + + elif isinstance(message, ToolMessage): + message_dict = { + "message_id": message.id, + "role": self._get_role(message), + "content": message.content, + "tool_call_id": message.tool_call_id, + } + + else: + message_dict = { + "message_id": message.id, + "role": self._get_role(message), + "content": message.content, + } + + messages_dict["messages"].append(message_dict) + messages_string = json.dumps(messages_dict) + else: + if "tools" in kwargs.keys(): + raise NotImplementedError( + "tool calling not supported in API Generic V2 without process_prompt, " + "switch to OpenAI compatible API or Generic V2 API with process_prompt=True" + ) messages_string = self.special_tokens["start"] for message in messages: messages_string += self.special_tokens["start_role"].format( @@ -1725,7 +1752,9 @@ def _handle_request( # create request payload for generic v1 API elif "api/v2/predict/generic" in self.sambastudio_url: - items = [{"id": "item0", "value": self._messages_to_string(messages)}] + items = [ + {"id": "item0", "value": self._messages_to_string(messages, **kwargs)} + ] params: Dict[str, Any] = { "select_expert": self.model, "process_prompt": self.process_prompt, @@ -1734,7 +1763,6 @@ def _handle_request( "top_p": self.top_p, "top_k": self.top_k, "do_sample": self.do_sample, - **kwargs, } if self.model_kwargs is not None: params = {**params, **self.model_kwargs} From 68515a38bd8147ab4149fc8d24ad2b43945c758b Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Tue, 19 Nov 2024 16:32:56 -0500 Subject: [PATCH 28/30] fmt --- .../langchain_community/chat_models/sambanova.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 99b9a431f52a3..fa8d2a6712df4 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -623,7 +623,8 @@ class AnswerWithJustification(BaseModel): if method == "function_calling": if schema is None: raise ValueError( - "`schema` must be specified when method is `function_calling`. Received None." + "`schema` must be specified when method is `function_calling`. " + "Received None." ) tool_name = convert_to_openai_tool(schema)["function"]["name"] llm = self.bind_tools([schema], tool_choice=tool_name) @@ -649,7 +650,8 @@ class AnswerWithJustification(BaseModel): elif method == "json_schema": if schema is None: raise ValueError( - "`schema` must be specified when method is not `json_mode`. Received None." + "`schema` must be specified when method is not `json_mode`. " + "Received None." ) llm = self # TODO bind response format when json schema available by API, @@ -1668,8 +1670,9 @@ def _messages_to_string(self, messages: List[BaseMessage], **kwargs: Any) -> str else: if "tools" in kwargs.keys(): raise NotImplementedError( - "tool calling not supported in API Generic V2 without process_prompt, " - "switch to OpenAI compatible API or Generic V2 API with process_prompt=True" + "tool calling not supported in API Generic V2 " + "without process_prompt, switch to OpenAI compatible API " + "or Generic V2 API with process_prompt=True" ) messages_string = self.special_tokens["start"] for message in messages: From 4f7039fea8e1d9d426c5f5a75cd049382042cbce Mon Sep 17 00:00:00 2001 From: Jorge Piedrahita Ortiz Date: Wed, 20 Nov 2024 09:28:51 -0500 Subject: [PATCH 29/30] fix typo --- libs/community/langchain_community/chat_models/sambanova.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index fa8d2a6712df4..01aa83cab952d 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -1753,7 +1753,7 @@ def _handle_request( **self.additional_headers, } - # create request payload for generic v1 API + # create request payload for generic v2 API elif "api/v2/predict/generic" in self.sambastudio_url: items = [ {"id": "item0", "value": self._messages_to_string(messages, **kwargs)} From 26810c7f4d50f20de0f4ca2891de1c54dc0286dc Mon Sep 17 00:00:00 2001 From: jhpiedrahitao Date: Wed, 4 Dec 2024 20:45:50 -0500 Subject: [PATCH 30/30] minor changes --- .../langchain_community/chat_models/sambanova.py | 8 ++++---- libs/community/langchain_community/llms/sambanova.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/libs/community/langchain_community/chat_models/sambanova.py b/libs/community/langchain_community/chat_models/sambanova.py index 17ac0a9b6c8ad..a2ddf24d8c8c0 100644 --- a/libs/community/langchain_community/chat_models/sambanova.py +++ b/libs/community/langchain_community/chat_models/sambanova.py @@ -1695,7 +1695,7 @@ def _get_sambastudio_urls(self, url: str) -> Tuple[str, str]: base_url: string with url to do non streaming calls streaming_url: string with url to do streaming calls """ - if "openai" in url: + if "chat/completions" in url: base_url = url stream_url = url else: @@ -1730,7 +1730,7 @@ def _handle_request( """ # create request payload for openai compatible API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: messages_dicts = _create_message_dicts(messages) data = { "messages": messages_dicts, @@ -1862,7 +1862,7 @@ def _process_response(self, response: Response) -> AIMessage: invalid_tool_calls = [] # process response payload for openai compatible API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: content = response_dict["choices"][0]["message"].get("content", "") if content is None: content = "" @@ -1956,7 +1956,7 @@ def _process_stream_response( ) # process response payload for openai compatible API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: finish_reason = "" client = sseclient.SSEClient(response) for event in client.events(): diff --git a/libs/community/langchain_community/llms/sambanova.py b/libs/community/langchain_community/llms/sambanova.py index 4e24b3889e653..994fb7888c875 100644 --- a/libs/community/langchain_community/llms/sambanova.py +++ b/libs/community/langchain_community/llms/sambanova.py @@ -175,7 +175,7 @@ def _get_sambastudio_urls(self, url: str) -> Tuple[str, str]: base_url: string with url to do non streaming calls streaming_url: string with url to do streaming calls """ - if "openai" in url: + if "chat/completions" in url: base_url = url stream_url = url else: @@ -214,7 +214,7 @@ def _get_tuning_params(self, stop: Optional[List[str]] = None) -> Dict[str, Any] _model_kwargs["stop_sequences"] = _stop_sequences # set the parameters structure depending of the API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: if "select_expert" in _model_kwargs.keys(): _model_kwargs["model"] = _model_kwargs.pop("select_expert") if "max_tokens_to_generate" in _model_kwargs.keys(): @@ -279,7 +279,7 @@ def _handle_request( params = self._get_tuning_params(stop) # create request payload for openAI v1 API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: messages_dict = [{"role": "user", "content": prompt[0]}] data = {"messages": messages_dict, "stream": streaming, **params} data = {key: value for key, value in data.items() if value is not None} @@ -378,7 +378,7 @@ def _process_response(self, response: Response) -> str: ) # process response payload for openai compatible API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: completion = response_dict["choices"][0]["message"]["content"] # process response payload for generic v2 API elif "api/v2/predict/generic" in self.sambastudio_url: @@ -413,7 +413,7 @@ def _process_stream_response(self, response: Response) -> Iterator[GenerationChu ) # process response payload for openai compatible API - if "openai" in self.sambastudio_url: + if "chat/completions" in self.sambastudio_url: client = sseclient.SSEClient(response) for event in client.events(): if event.event == "error_event":