From f58cf122d57348fb574431e579c62794a1e14155 Mon Sep 17 00:00:00 2001 From: jlqueguiner Date: Mon, 16 Oct 2023 22:29:17 +0200 Subject: [PATCH 01/10] feat(llm): add together.xyz provider --- libs/langchain/langchain/llms/__init__.py | 9 + libs/langchain/langchain/llms/together.py | 213 ++++++++++++++++++ .../integration_tests/llms/test_together.py | 26 +++ 3 files changed, 248 insertions(+) create mode 100644 libs/langchain/langchain/llms/together.py create mode 100644 libs/langchain/tests/integration_tests/llms/test_together.py diff --git a/libs/langchain/langchain/llms/__init__.py b/libs/langchain/langchain/llms/__init__.py index b4d74f470ac1e..835b7323daada 100644 --- a/libs/langchain/langchain/llms/__init__.py +++ b/libs/langchain/langchain/llms/__init__.py @@ -432,6 +432,12 @@ def _import_titan_takeoff() -> Any: return TitanTakeoff +def _import_together() -> Any: + from langchain.llms.together import Together + + return Together + + def _import_tongyi() -> Any: from langchain.llms.tongyi import Tongyi @@ -611,6 +617,8 @@ def __getattr__(name: str) -> Any: return _import_textgen() elif name == "TitanTakeoff": return _import_titan_takeoff() + elif name == "Together": + return _import_together() elif name == "Tongyi": return _import_tongyi() elif name == "VertexAI": @@ -773,6 +781,7 @@ def get_type_to_cls_dict() -> Dict[str, Callable[[], Type[BaseLLM]]]: "self_hosted": _import_self_hosted, "self_hosted_hugging_face": _import_self_hosted_hugging_face, "stochasticai": _import_stochasticai, + "together": _import_together, "tongyi": _import_tongyi, "titan_takeoff": _import_titan_takeoff, "vertexai": _import_vertex, diff --git a/libs/langchain/langchain/llms/together.py b/libs/langchain/langchain/llms/together.py new file mode 100644 index 0000000000000..8945ce7af432e --- /dev/null +++ b/libs/langchain/langchain/llms/together.py @@ -0,0 +1,213 @@ +"""Wrapper around Together's Generation API.""" +import logging +from typing import Any, Dict, List, Literal, Optional + +from aiohttp import ClientSession + +from langchain.callbacks.manager import ( + AsyncCallbackManagerForLLMRun, + CallbackManagerForLLMRun, +) +from langchain.llms.base import LLM +from langchain.llms.utils import enforce_stop_tokens +from langchain.pydantic_v1 import Extra, Field, root_validator +from langchain.utilities.requests import Requests +from langchain.utils import get_from_dict_or_env + +logger = logging.getLogger(__name__) + + +class Together(LLM): + """Wrapper around together models. + + To use, you should have + the environment variable ``TOGETHER_API_KEY`` set with your API token. + You can find your token here: https://api.together.xyz/settings/api-keys + + + params format { + "model": "togethercomputer/RedPajama-INCITE-7B-Instruct", + "prompt": "Q: The capital of France is?\nA:", + "temperature": 0.7, + "top_p": 0.7, + "top_k": 50, + "max_tokens": 1, + "repetition_penalty": 1 + } + + for api reference check together documentation: https://docs.together.ai/docs/inference-rest. + """ + + base_url: str = "https://api.together.xyz/inference" + + together_api_key: Optional[str] = None + + model: Optional[str] = None + """ + model name for above provider (eg: 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' for RedPajama-INCITE Chat (3B)) + available models are shown on https://docs.together.ai/docs/inference-models under 'available providers' + """ + + temperature: Optional[float] = Field(default=0.7, ge=0, le=1) # for text + top_p: Optional[float] = Field(default=0.7, ge=0, le=1) # for text + top_k: Optional[int] = Field(default=50, ge=0) # for text + max_tokens: Optional[int] = Field(default=128, ge=0) # for text + repetition_penalty: Optional[float] = Field(default=1, ge=0, le=1) # for text + + + class Config: + """Configuration for this pydantic object.""" + extra = Extra.forbid + + @root_validator() + def validate_environment(cls, values: Dict) -> Dict: + """Validate that api key exists in environment.""" + values["together_api_key"] = get_from_dict_or_env( + values, "together_api_key", "TOGETHER_API_KEY" + ) + return values + + @property + def _llm_type(self) -> str: + """Return type of model.""" + return "together" + + def _format_output(self, output: dict) -> str: + return output["output"]["choices"][0]["text"] + + @staticmethod + def get_user_agent() -> str: + from langchain import __version__ + + return f"langchain/{__version__}" + + def _call( + self, + prompt: str, + stop: Optional[List[str]] = None, + run_manager: Optional[CallbackManagerForLLMRun] = None, + **kwargs: Any, + ) -> str: + """Call out to Together's text generation endpoint. + + Args: + prompt: The prompt to pass into the model. + + Returns: + json formatted str response. + """ + + url = f"{self.base_url}" + headers = { + "Authorization": f"Bearer {self.together_api_key}", + "Content-Type": "application/json" + } + payload: Dict[str, Any] = { + "model": self.model, + "prompt": prompt, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "max_tokens": self.max_tokens, + "repetition_penalty": self.repetition_penalty + } + + # filter None values to not pass them to the http payload + payload = {k: v for k, v in payload.items() if v is not None} + + request = Requests(headers=headers) + response = request.post(url=url, data=payload) + + if response.status_code >= 500: + raise Exception(f"Together Server: Error {response.status_code}") + elif response.status_code >= 400: + raise ValueError(f"Together received an invalid payload: {response.text}") + elif response.status_code != 200: + raise Exception( + f"Together returned an unexpected response with status " + f"{response.status_code}: {response.text}" + ) + + data = response.json() + + if data.get("status") != "finished": + err_msg = data.get("error", "Undefined Error") + raise Exception(err_msg) + + output = self._format_output(data) + + + return output + + async def _acall( + self, + prompt: str, + stop: Optional[List[str]] = None, + run_manager: Optional[AsyncCallbackManagerForLLMRun] = None, + **kwargs: Any, + ) -> str: + """Call Together model to get predictions based on the prompt. + + Args: + prompt: The prompt to pass into the model. + + Returns: + The string generated by the model. + """ + + stops = None + if self.stop_sequences is not None and stop is not None: + raise ValueError( + "stop sequences found in both the input and default params." + ) + elif self.stop_sequences is not None: + stops = self.stop_sequences + else: + stops = stop + + headers = { + "Authorization": f"Bearer {self.together_api_key}", + "Content-Type": "application/json" + } + + payload: Dict[str, Any] = { + "model": self.model, + "prompt": prompt, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "max_tokens": self.max_tokens, + "repetition_penalty": self.repetition_penalty + } + + # filter `None` values to not pass them to the http payload as null + payload = {k: v for k, v in payload.items() if v is not None} + + if self.model is not None: + payload["settings"] = {self.provider: self.model} + + async with ClientSession() as session: + async with session.post(self.base_url, json=payload, headers=headers) as response: + if response.status >= 500: + raise Exception(f"Together Server: Error {response.status}") + elif response.status >= 400: + raise ValueError( + f"Together received an invalid payload: {response.text}" + ) + elif response.status != 200: + raise Exception( + f"Together returned an unexpected response with status " + f"{response.status}: {response.text}" + ) + + response_json = await response.json() + + if response_json.get("status") != "finished": + err_msg = response_json.get("error", "Undefined Error") + raise Exception(err_msg) + + output = self._format_output(response_json) + if stops is not None: + output = enforce_stop_tokens(output, stops) + + return output diff --git a/libs/langchain/tests/integration_tests/llms/test_together.py b/libs/langchain/tests/integration_tests/llms/test_together.py new file mode 100644 index 0000000000000..11024552ae658 --- /dev/null +++ b/libs/langchain/tests/integration_tests/llms/test_together.py @@ -0,0 +1,26 @@ +"""Test Together API wrapper. + +In order to run this test, you need to have an Together api key. +You can get it by registering for free at https://api.together.xyz/. +A test key can be found at https://api.together.xyz/settings/api-keys + +You'll then need to set TOGETHER_API_KEY environment variable to your api key. +""" +from langchain.llms import Together + + +def test_together_call() -> None: + """Test simple call to together.""" + llm = Together(temperature=0.2, max_tokens=250) + output = llm("Say foo:") + + assert llm._llm_type == "together" + assert isinstance(output, str) + + +async def test_together_acall() -> None: + """Test simple call to together.""" + output = await llm.agenerate(["Say foo:"]) + llm = Together(temperature=0.2, max_tokens=250) + assert llm._llm_type == "together" + assert isinstance(output, str) From 9a841e97a3e4f61f94a5439f8ab9f4d3f0345bd6 Mon Sep 17 00:00:00 2001 From: jlqueguiner Date: Mon, 16 Oct 2023 22:46:53 +0200 Subject: [PATCH 02/10] fix(ruff): fix many ruff issues --- libs/langchain/langchain/llms/together.py | 31 ++++++++++--------- .../prompts/example_selector/ngram_overlap.py | 23 +++++++++----- .../integration_tests/llms/test_together.py | 3 +- .../unit_tests/chains/test_llm_checker.py | 8 ++--- 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/libs/langchain/langchain/llms/together.py b/libs/langchain/langchain/llms/together.py index 8945ce7af432e..9925f83c91c3f 100644 --- a/libs/langchain/langchain/llms/together.py +++ b/libs/langchain/langchain/llms/together.py @@ -1,6 +1,6 @@ """Wrapper around Together's Generation API.""" import logging -from typing import Any, Dict, List, Literal, Optional +from typing import Any, Dict, List, Optional from aiohttp import ClientSession @@ -24,7 +24,7 @@ class Together(LLM): the environment variable ``TOGETHER_API_KEY`` set with your API token. You can find your token here: https://api.together.xyz/settings/api-keys - + params format { "model": "togethercomputer/RedPajama-INCITE-7B-Instruct", "prompt": "Q: The capital of France is?\nA:", @@ -35,7 +35,8 @@ class Together(LLM): "repetition_penalty": 1 } - for api reference check together documentation: https://docs.together.ai/docs/inference-rest. + for api reference check together documentation: + https://docs.together.ai/docs/inference-rest """ base_url: str = "https://api.together.xyz/inference" @@ -44,8 +45,9 @@ class Together(LLM): model: Optional[str] = None """ - model name for above provider (eg: 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' for RedPajama-INCITE Chat (3B)) - available models are shown on https://docs.together.ai/docs/inference-models under 'available providers' + model name for above provider (eg: 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' + for RedPajama-INCITE Chat (3B)) + available models are shown on https://docs.together.ai/docs/inference-models """ temperature: Optional[float] = Field(default=0.7, ge=0, le=1) # for text @@ -53,10 +55,10 @@ class Together(LLM): top_k: Optional[int] = Field(default=50, ge=0) # for text max_tokens: Optional[int] = Field(default=128, ge=0) # for text repetition_penalty: Optional[float] = Field(default=1, ge=0, le=1) # for text - class Config: """Configuration for this pydantic object.""" + extra = Extra.forbid @root_validator() @@ -100,7 +102,7 @@ def _call( url = f"{self.base_url}" headers = { "Authorization": f"Bearer {self.together_api_key}", - "Content-Type": "application/json" + "Content-Type": "application/json", } payload: Dict[str, Any] = { "model": self.model, @@ -109,7 +111,7 @@ def _call( "top_p": self.top_p, "top_k": self.top_k, "max_tokens": self.max_tokens, - "repetition_penalty": self.repetition_penalty + "repetition_penalty": self.repetition_penalty, } # filter None values to not pass them to the http payload @@ -129,14 +131,13 @@ def _call( ) data = response.json() - + if data.get("status") != "finished": err_msg = data.get("error", "Undefined Error") raise Exception(err_msg) output = self._format_output(data) - return output async def _acall( @@ -167,7 +168,7 @@ async def _acall( headers = { "Authorization": f"Bearer {self.together_api_key}", - "Content-Type": "application/json" + "Content-Type": "application/json", } payload: Dict[str, Any] = { @@ -177,7 +178,7 @@ async def _acall( "top_p": self.top_p, "top_k": self.top_k, "max_tokens": self.max_tokens, - "repetition_penalty": self.repetition_penalty + "repetition_penalty": self.repetition_penalty, } # filter `None` values to not pass them to the http payload as null @@ -187,7 +188,9 @@ async def _acall( payload["settings"] = {self.provider: self.model} async with ClientSession() as session: - async with session.post(self.base_url, json=payload, headers=headers) as response: + async with session.post( + self.base_url, json=payload, headers=headers + ) as response: if response.status >= 500: raise Exception(f"Together Server: Error {response.status}") elif response.status >= 400: @@ -201,7 +204,7 @@ async def _acall( ) response_json = await response.json() - + if response_json.get("status") != "finished": err_msg = response_json.get("error", "Undefined Error") raise Exception(err_msg) diff --git a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py index 969c81e43d6b5..7576117038747 100644 --- a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py +++ b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py @@ -3,6 +3,7 @@ https://www.nltk.org/_modules/nltk/translate/bleu_score.html https://aclanthology.org/P02-1040.pdf """ +import importlib.util from typing import Dict, List import numpy as np @@ -66,16 +67,24 @@ class NGramOverlapExampleSelector(BaseExampleSelector, BaseModel): @root_validator(pre=True) def check_dependencies(cls, values: Dict) -> Dict: """Check that valid dependencies exist.""" - try: - from nltk.translate.bleu_score import ( # noqa: disable=F401 - SmoothingFunction, - sentence_bleu, - ) - except ImportError as e: + spec = importlib.util.find_spec("nltk.translate.bleu_score") + if spec is None: raise ImportError( "Not all the correct dependencies for this ExampleSelect exist." "Please install nltk with `pip install nltk`." - ) from e + ) + + # Module exists, now we need to check for the specific attributes + nltk_bleu = importlib.import_module("nltk.translate.bleu_score") + if ( + not hasattr(nltk_bleu, "sentence_bleu") or + not hasattr(nltk_bleu, "SmoothingFunction") + ): + raise ImportError( + "Required functions or classes are missing in nltk's bleu_score module. " # noqa: E501 + "Please ensure you have the correct version of nltk." + ) + return values diff --git a/libs/langchain/tests/integration_tests/llms/test_together.py b/libs/langchain/tests/integration_tests/llms/test_together.py index 11024552ae658..dafe4374eab01 100644 --- a/libs/langchain/tests/integration_tests/llms/test_together.py +++ b/libs/langchain/tests/integration_tests/llms/test_together.py @@ -20,7 +20,8 @@ def test_together_call() -> None: async def test_together_acall() -> None: """Test simple call to together.""" - output = await llm.agenerate(["Say foo:"]) llm = Together(temperature=0.2, max_tokens=250) + output = await llm.agenerate(["Say foo:"]) + assert llm._llm_type == "together" assert isinstance(output, str) diff --git a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py index cc2ceb9909c2b..744c191cac1b3 100644 --- a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py +++ b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py @@ -23,12 +23,12 @@ def fake_llm_checker_chain() -> LLMCheckerChain: ): "I don't know which mammal layers the biggest eggs.", _LIST_ASSERTIONS_TEMPLATE.format( statement="I don't know which mammal layers the biggest eggs.", - ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", + ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 _CHECK_ASSERTIONS_TEMPLATE.format( - assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", - ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", + assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 + ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 _REVISED_ANSWER_TEMPLATE.format( - checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", + checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 question="Which mammal lays the biggest eggs?", ): "I still don't know.", } From f320a988a4b5f8f22ed5abe8b95301bf3a256eea Mon Sep 17 00:00:00 2001 From: Jean-Louis Queguiner Date: Mon, 16 Oct 2023 23:46:07 +0200 Subject: [PATCH 03/10] fix(llm): fix together overwrite payload with kwargs --- libs/langchain/langchain/llms/together.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/libs/langchain/langchain/llms/together.py b/libs/langchain/langchain/llms/together.py index 9925f83c91c3f..f4a8ec605398f 100644 --- a/libs/langchain/langchain/llms/together.py +++ b/libs/langchain/langchain/llms/together.py @@ -35,7 +35,7 @@ class Together(LLM): "repetition_penalty": 1 } - for api reference check together documentation: + for api reference check together documentation: https://docs.together.ai/docs/inference-rest """ @@ -114,6 +114,17 @@ def _call( "repetition_penalty": self.repetition_penalty, } + # Overwrite the values in payload with kwargs if they are provided + for key in [ + "temperature", + "top_p", + "top_k", + "max_tokens", + "repetition_penalty", + ]: + if key in kwargs: + payload[key] = kwargs[key] + # filter None values to not pass them to the http payload payload = {k: v for k, v in payload.items() if v is not None} From e64b489876636a376315e5b24a6b0a82522f7122 Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:21:31 -0700 Subject: [PATCH 04/10] cr --- libs/langchain/langchain/llms/together.py | 143 ++++++++---------- .../prompts/example_selector/ngram_overlap.py | 8 +- .../integration_tests/llms/test_together.py | 23 ++- .../unit_tests/chains/test_llm_checker.py | 8 +- 4 files changed, 86 insertions(+), 96 deletions(-) diff --git a/libs/langchain/langchain/llms/together.py b/libs/langchain/langchain/llms/together.py index f4a8ec605398f..c5445c1ad0cda 100644 --- a/libs/langchain/langchain/llms/together.py +++ b/libs/langchain/langchain/llms/together.py @@ -1,4 +1,4 @@ -"""Wrapper around Together's Generation API.""" +"""Wrapper around Together AI's Completion API.""" import logging from typing import Any, Dict, List, Optional @@ -9,8 +9,7 @@ CallbackManagerForLLMRun, ) from langchain.llms.base import LLM -from langchain.llms.utils import enforce_stop_tokens -from langchain.pydantic_v1 import Extra, Field, root_validator +from langchain.pydantic_v1 import Extra, root_validator from langchain.utilities.requests import Requests from langchain.utils import get_from_dict_or_env @@ -18,50 +17,56 @@ class Together(LLM): - """Wrapper around together models. + """Wrapper around Together AI models. - To use, you should have - the environment variable ``TOGETHER_API_KEY`` set with your API token. - You can find your token here: https://api.together.xyz/settings/api-keys + To use, you'll need an API key which you can find here: + https://api.together.xyz/settings/api-keys. This can be passed in as init param + ``together_api_key`` or set as environment variable ``TOGETHER_API_KEY``. - - params format { - "model": "togethercomputer/RedPajama-INCITE-7B-Instruct", - "prompt": "Q: The capital of France is?\nA:", - "temperature": 0.7, - "top_p": 0.7, - "top_k": 50, - "max_tokens": 1, - "repetition_penalty": 1 - } - - for api reference check together documentation: - https://docs.together.ai/docs/inference-rest + Together AI API reference: https://docs.together.ai/reference/inference """ base_url: str = "https://api.together.xyz/inference" - - together_api_key: Optional[str] = None - - model: Optional[str] = None + """Base inference API URL.""" + together_api_key: str + """Together AI API key. Get it here: https://api.together.xyz/settings/api-keys""" + model: str + """Model name. Available models listed here: + https://docs.together.ai/docs/inference-models """ - model name for above provider (eg: 'togethercomputer/RedPajama-INCITE-Chat-3B-v1' - for RedPajama-INCITE Chat (3B)) - available models are shown on https://docs.together.ai/docs/inference-models + temperature: Optional[float] = None + """Model temperature.""" + top_p: Optional[float] = None + """Used to dynamically adjust the number of choices for each predicted token based + on the cumulative probabilities. A value of 1 will always yield the same + output. A temperature less than 1 favors more correctness and is appropriate + for question answering or summarization. A value greater than 1 introduces more + randomness in the output. + """ + top_k: Optional[int] = None + """Used to limit the number of choices for the next predicted word or token. It + specifies the maximum number of tokens to consider at each step, based on their + probability of occurrence. This technique helps to speed up the generation + process and can improve the quality of the generated text by focusing on the + most likely options. + """ + max_tokens: Optional[int] = None + """The maximum number of tokens to generate.""" + repetition_penalty: Optional[float] = None + """A number that controls the diversity of generated text by reducing the + likelihood of repeated sequences. Higher values decrease repetition. + """ + logprobs: Optional[int] = None + """An integer that specifies how many top token log probabilities are included in + the response for each token generation step. """ - - temperature: Optional[float] = Field(default=0.7, ge=0, le=1) # for text - top_p: Optional[float] = Field(default=0.7, ge=0, le=1) # for text - top_k: Optional[int] = Field(default=50, ge=0) # for text - max_tokens: Optional[int] = Field(default=128, ge=0) # for text - repetition_penalty: Optional[float] = Field(default=1, ge=0, le=1) # for text class Config: """Configuration for this pydantic object.""" extra = Extra.forbid - @root_validator() + @root_validator(pre=True) def validate_environment(cls, values: Dict) -> Dict: """Validate that api key exists in environment.""" values["together_api_key"] = get_from_dict_or_env( @@ -83,6 +88,17 @@ def get_user_agent() -> str: return f"langchain/{__version__}" + @property + def default_params(self) -> Dict[str, Any]: + return { + "model": self.model, + "temperature": self.temperature, + "top_p": self.top_p, + "top_k": self.top_k, + "max_tokens": self.max_tokens, + "repetition_penalty": self.repetition_penalty, + } + def _call( self, prompt: str, @@ -96,40 +112,25 @@ def _call( prompt: The prompt to pass into the model. Returns: - json formatted str response. + The string generated by the model.. """ - url = f"{self.base_url}" headers = { "Authorization": f"Bearer {self.together_api_key}", "Content-Type": "application/json", } + stop_to_use = stop[0] if stop and len(stop) == 1 else stop payload: Dict[str, Any] = { - "model": self.model, + **self.default_params, "prompt": prompt, - "temperature": self.temperature, - "top_p": self.top_p, - "top_k": self.top_k, - "max_tokens": self.max_tokens, - "repetition_penalty": self.repetition_penalty, + "stop": stop_to_use, + **kwargs, } - # Overwrite the values in payload with kwargs if they are provided - for key in [ - "temperature", - "top_p", - "top_k", - "max_tokens", - "repetition_penalty", - ]: - if key in kwargs: - payload[key] = kwargs[key] - # filter None values to not pass them to the http payload payload = {k: v for k, v in payload.items() if v is not None} - request = Requests(headers=headers) - response = request.post(url=url, data=payload) + response = request.post(url=self.base_url, data=payload) if response.status_code >= 500: raise Exception(f"Together Server: Error {response.status_code}") @@ -142,7 +143,6 @@ def _call( ) data = response.json() - if data.get("status") != "finished": err_msg = data.get("error", "Undefined Error") raise Exception(err_msg) @@ -166,38 +166,20 @@ async def _acall( Returns: The string generated by the model. """ - - stops = None - if self.stop_sequences is not None and stop is not None: - raise ValueError( - "stop sequences found in both the input and default params." - ) - elif self.stop_sequences is not None: - stops = self.stop_sequences - else: - stops = stop - headers = { "Authorization": f"Bearer {self.together_api_key}", "Content-Type": "application/json", } - + stop_to_use = stop[0] if stop and len(stop) == 1 else stop payload: Dict[str, Any] = { - "model": self.model, + **self.default_params, "prompt": prompt, - "temperature": self.temperature, - "top_p": self.top_p, - "top_k": self.top_k, - "max_tokens": self.max_tokens, - "repetition_penalty": self.repetition_penalty, + "stop": stop_to_use, + **kwargs, } - # filter `None` values to not pass them to the http payload as null + # filter None values to not pass them to the http payload payload = {k: v for k, v in payload.items() if v is not None} - - if self.model is not None: - payload["settings"] = {self.provider: self.model} - async with ClientSession() as session: async with session.post( self.base_url, json=payload, headers=headers @@ -221,7 +203,4 @@ async def _acall( raise Exception(err_msg) output = self._format_output(response_json) - if stops is not None: - output = enforce_stop_tokens(output, stops) - return output diff --git a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py index 7576117038747..858d1360d0735 100644 --- a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py +++ b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py @@ -76,16 +76,14 @@ def check_dependencies(cls, values: Dict) -> Dict: # Module exists, now we need to check for the specific attributes nltk_bleu = importlib.import_module("nltk.translate.bleu_score") - if ( - not hasattr(nltk_bleu, "sentence_bleu") or - not hasattr(nltk_bleu, "SmoothingFunction") + if not hasattr(nltk_bleu, "sentence_bleu") or not hasattr( + nltk_bleu, "SmoothingFunction" ): raise ImportError( - "Required functions or classes are missing in nltk's bleu_score module. " # noqa: E501 + "Required functions or classes are missing in nltk's bleu_score module. " # noqa: E501 "Please ensure you have the correct version of nltk." ) - return values def add_example(self, example: Dict[str, str]) -> None: diff --git a/libs/langchain/tests/integration_tests/llms/test_together.py b/libs/langchain/tests/integration_tests/llms/test_together.py index dafe4374eab01..ca88c6c8614c7 100644 --- a/libs/langchain/tests/integration_tests/llms/test_together.py +++ b/libs/langchain/tests/integration_tests/llms/test_together.py @@ -6,22 +6,35 @@ You'll then need to set TOGETHER_API_KEY environment variable to your api key. """ +import pytest as pytest + from langchain.llms import Together def test_together_call() -> None: """Test simple call to together.""" - llm = Together(temperature=0.2, max_tokens=250) + llm = Together( + model="togethercomputer/RedPajama-INCITE-7B-Base", + temperature=0.2, + max_tokens=250, + ) output = llm("Say foo:") assert llm._llm_type == "together" assert isinstance(output, str) +@pytest.mark.asyncio async def test_together_acall() -> None: """Test simple call to together.""" - llm = Together(temperature=0.2, max_tokens=250) - output = await llm.agenerate(["Say foo:"]) - + llm = Together( + model="togethercomputer/RedPajama-INCITE-7B-Base", + temperature=0.2, + max_tokens=250, + ) + output = await llm.agenerate(["Say foo:"], stop=["bar"]) + assert llm._llm_type == "together" - assert isinstance(output, str) + output_text = output.generations[0][0].text + assert isinstance(output_text, str) + assert output_text.count("bar") <= 1 diff --git a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py index 744c191cac1b3..ede6ef4487ec1 100644 --- a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py +++ b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py @@ -23,12 +23,12 @@ def fake_llm_checker_chain() -> LLMCheckerChain: ): "I don't know which mammal layers the biggest eggs.", _LIST_ASSERTIONS_TEMPLATE.format( statement="I don't know which mammal layers the biggest eggs.", - ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 + ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 _CHECK_ASSERTIONS_TEMPLATE.format( - assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 - ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 + assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 + ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 _REVISED_ANSWER_TEMPLATE.format( - checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 + checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 question="Which mammal lays the biggest eggs?", ): "I still don't know.", } From afbf36872c094f25e10bce73dbd4629e25a5bf2a Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:21:58 -0700 Subject: [PATCH 05/10] undo --- .../prompts/example_selector/ngram_overlap.py | 21 +++++++------------ .../unit_tests/chains/test_llm_checker.py | 8 +++---- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py index 858d1360d0735..969c81e43d6b5 100644 --- a/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py +++ b/libs/langchain/langchain/prompts/example_selector/ngram_overlap.py @@ -3,7 +3,6 @@ https://www.nltk.org/_modules/nltk/translate/bleu_score.html https://aclanthology.org/P02-1040.pdf """ -import importlib.util from typing import Dict, List import numpy as np @@ -67,22 +66,16 @@ class NGramOverlapExampleSelector(BaseExampleSelector, BaseModel): @root_validator(pre=True) def check_dependencies(cls, values: Dict) -> Dict: """Check that valid dependencies exist.""" - spec = importlib.util.find_spec("nltk.translate.bleu_score") - if spec is None: + try: + from nltk.translate.bleu_score import ( # noqa: disable=F401 + SmoothingFunction, + sentence_bleu, + ) + except ImportError as e: raise ImportError( "Not all the correct dependencies for this ExampleSelect exist." "Please install nltk with `pip install nltk`." - ) - - # Module exists, now we need to check for the specific attributes - nltk_bleu = importlib.import_module("nltk.translate.bleu_score") - if not hasattr(nltk_bleu, "sentence_bleu") or not hasattr( - nltk_bleu, "SmoothingFunction" - ): - raise ImportError( - "Required functions or classes are missing in nltk's bleu_score module. " # noqa: E501 - "Please ensure you have the correct version of nltk." - ) + ) from e return values diff --git a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py index ede6ef4487ec1..cc2ceb9909c2b 100644 --- a/libs/langchain/tests/unit_tests/chains/test_llm_checker.py +++ b/libs/langchain/tests/unit_tests/chains/test_llm_checker.py @@ -23,12 +23,12 @@ def fake_llm_checker_chain() -> LLMCheckerChain: ): "I don't know which mammal layers the biggest eggs.", _LIST_ASSERTIONS_TEMPLATE.format( statement="I don't know which mammal layers the biggest eggs.", - ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 + ): "1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", _CHECK_ASSERTIONS_TEMPLATE.format( - assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", # noqa: E501 - ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 + assertions="1) I know that mammals lay eggs.\n2) I know that birds lay eggs.\n3) I know that birds are mammals.", + ): "1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", _REVISED_ANSWER_TEMPLATE.format( - checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", # noqa: E501 + checked_assertions="1) I know that mammals lay eggs. TRUE\n2) I know that birds lay eggs. TRUE\n3) I know that birds are mammals. TRUE", question="Which mammal lays the biggest eggs?", ): "I still don't know.", } From 1dafd131a78d6d333990ccf003e0f5dc780de512 Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:35:16 -0700 Subject: [PATCH 06/10] nb --- docs/docs/integrations/llms/ollama.ipynb | 2 +- docs/docs/integrations/llms/together.ipynb | 87 ++++++++++++++++++++++ 2 files changed, 88 insertions(+), 1 deletion(-) create mode 100644 docs/docs/integrations/llms/together.ipynb diff --git a/docs/docs/integrations/llms/ollama.ipynb b/docs/docs/integrations/llms/ollama.ipynb index e395603138df0..5e18c048f4b6a 100644 --- a/docs/docs/integrations/llms/ollama.ipynb +++ b/docs/docs/integrations/llms/ollama.ipynb @@ -417,7 +417,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.16" + "version": "3.9.1" } }, "nbformat": 4, diff --git a/docs/docs/integrations/llms/together.ipynb b/docs/docs/integrations/llms/together.ipynb new file mode 100644 index 0000000000000..bf67e925769f0 --- /dev/null +++ b/docs/docs/integrations/llms/together.ipynb @@ -0,0 +1,87 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "2970dd75-8ebf-4b51-8282-9b454b8f356d", + "metadata": {}, + "source": [ + "# [Together AI](https://together.ai)\n", + "\n", + "> The Together API makes it easy to fine-tune or run leading open-source models with a couple lines of code. We have integrated the world’s leading open-source models, including Llama-2, RedPajama, Falcon, Alpaca, Stable Diffusion XL, and more.\n", + "\n", + "To use, you'll need an API key which you can find here:\n", + "https://api.together.xyz/settings/api-keys. This can be passed in as init param\n", + "``together_api_key`` or set as environment variable ``TOGETHER_API_KEY``.\n", + "\n", + "Together API reference: https://docs.together.ai/reference/inference" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "e7b7170d-d7c5-4890-9714-a37238343805", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "A: A large language model is a neural network that is trained on a large amount of text data. It is able to generate text that is similar to the training data, and can be used for tasks such as language translation, question answering, and text summarization.\n", + "\n", + "A: A large language model is a neural network that is trained on a large amount of text data. It is able to generate text that is similar to the training data, and can be used for tasks such as language translation, question answering, and text summarization.\n", + "\n", + "A: A large language model is a neural network that is trained on\n" + ] + } + ], + "source": [ + "from langchain.llms import Together\n", + "\n", + "llm = Together(\n", + " model=\"togethercomputer/RedPajama-INCITE-7B-Base\",\n", + " temperature=0.7,\n", + " max_tokens=128,\n", + " top_k=1,\n", + " together_api_key=\"6c997d123263c65da959ee3d664d81821d391acef81a8ec1df80ef267b1aeecd\"\n", + ")\n", + "\n", + "input_ = \"\"\"You are a teacher with a deep knowledge of machine learning and AI. \\\n", + "You provide succinct and accurate answers. Answer the following question: \n", + "\n", + "What is a large language model?\"\"\"\n", + "print(llm(input_))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ce94c6de-5258-46c1-89f9-88fd3bcc10c1", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "poetry-venv", + "language": "python", + "name": "poetry-venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 6c0bc3d2432881aaf86caa9b31e24c0e30d47a1f Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:35:28 -0700 Subject: [PATCH 07/10] nit --- docs/docs/integrations/llms/together.ipynb | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/docs/integrations/llms/together.ipynb b/docs/docs/integrations/llms/together.ipynb index bf67e925769f0..28d593c171de3 100644 --- a/docs/docs/integrations/llms/together.ipynb +++ b/docs/docs/integrations/llms/together.ipynb @@ -53,14 +53,6 @@ "What is a large language model?\"\"\"\n", "print(llm(input_))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ce94c6de-5258-46c1-89f9-88fd3bcc10c1", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From 65e580de3690196dbbef8b3bbc2e6e621ec4ed36 Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:35:59 -0700 Subject: [PATCH 08/10] undo --- docs/docs/integrations/llms/ollama.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/integrations/llms/ollama.ipynb b/docs/docs/integrations/llms/ollama.ipynb index 5e18c048f4b6a..e395603138df0 100644 --- a/docs/docs/integrations/llms/ollama.ipynb +++ b/docs/docs/integrations/llms/ollama.ipynb @@ -417,7 +417,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.1" + "version": "3.9.16" } }, "nbformat": 4, From 3bb54713e3df887103b42ac0b667d61eb667365a Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 16:41:44 -0700 Subject: [PATCH 09/10] undo --- docs/docs/integrations/llms/together.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/integrations/llms/together.ipynb b/docs/docs/integrations/llms/together.ipynb index 28d593c171de3..650d2303cf627 100644 --- a/docs/docs/integrations/llms/together.ipynb +++ b/docs/docs/integrations/llms/together.ipynb @@ -44,7 +44,7 @@ " temperature=0.7,\n", " max_tokens=128,\n", " top_k=1,\n", - " together_api_key=\"6c997d123263c65da959ee3d664d81821d391acef81a8ec1df80ef267b1aeecd\"\n", + " # together_api_key=\"...\"\n", ")\n", "\n", "input_ = \"\"\"You are a teacher with a deep knowledge of machine learning and AI. \\\n", From 758df1c3c5f94f9e31107e9dc248fbf4fb99d8ff Mon Sep 17 00:00:00 2001 From: Bagatur Date: Mon, 16 Oct 2023 17:07:23 -0700 Subject: [PATCH 10/10] nit --- docs/docs/integrations/llms/together.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/integrations/llms/together.ipynb b/docs/docs/integrations/llms/together.ipynb index 650d2303cf627..d6e89e4c1b2c5 100644 --- a/docs/docs/integrations/llms/together.ipynb +++ b/docs/docs/integrations/llms/together.ipynb @@ -5,9 +5,9 @@ "id": "2970dd75-8ebf-4b51-8282-9b454b8f356d", "metadata": {}, "source": [ - "# [Together AI](https://together.ai)\n", + "# Together AI\n", "\n", - "> The Together API makes it easy to fine-tune or run leading open-source models with a couple lines of code. We have integrated the world’s leading open-source models, including Llama-2, RedPajama, Falcon, Alpaca, Stable Diffusion XL, and more.\n", + "> The Together API makes it easy to fine-tune or run leading open-source models with a couple lines of code. We have integrated the world’s leading open-source models, including Llama-2, RedPajama, Falcon, Alpaca, Stable Diffusion XL, and more. Read more: https://together.ai\n", "\n", "To use, you'll need an API key which you can find here:\n", "https://api.together.xyz/settings/api-keys. This can be passed in as init param\n",