From 300204efdeb1b20f3f3a015d2060456d5517c998 Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 10:03:47 +0200 Subject: [PATCH 1/6] add gpt4generator --- .../components/generators/openai/gpt4.py | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 haystack/preview/components/generators/openai/gpt4.py diff --git a/haystack/preview/components/generators/openai/gpt4.py b/haystack/preview/components/generators/openai/gpt4.py new file mode 100644 index 0000000000..f258f9c5d0 --- /dev/null +++ b/haystack/preview/components/generators/openai/gpt4.py @@ -0,0 +1,70 @@ +from typing import Optional, Callable + +import logging + +from haystack.preview import component + + +logger = logging.getLogger(__name__) + + +@component +class GPT4Generator: + """ + LLM Generator compatible with GPT4 large language models. + + Queries the LLM using OpenAI's API. Invocations are made using OpenAI SDK ('openai' package) + See [OpenAI GPT4 API](https://platform.openai.com/docs/guides/chat) for more details. + """ + + def __init__( + self, + api_key: str, + model_name: str = "gpt-4", + system_prompt: Optional[str] = None, + streaming_callback: Optional[Callable] = None, + api_base_url: str = "https://api.openai.com/v1", + **kwargs, + ): + """ + Creates an instance of GPT4Generator for OpenAI's GPT-4 model. + + :param api_key: The OpenAI API key. + :param model_name: The name of the model to use. + :param system_prompt: An additional message to be sent to the LLM at the beginning of each conversation. + Typically, a conversation is formatted with a system message first, followed by alternating messages from + the 'user' (the "queries") and the 'assistant' (the "responses"). The system message helps set the behavior + of the assistant. For example, you can modify the personality of the assistant or provide specific + instructions about how it should behave throughout the conversation. + :param streaming_callback: A callback function that is called when a new token is received from the stream. + The callback function should accept two parameters: the token received from the stream and **kwargs. + The callback function should return the token to be sent to the stream. If the callback function is not + provided, the token is printed to stdout. + :param api_base_url: The OpenAI API Base url, defaults to `https://api.openai.com/v1`. + :param kwargs: Other parameters to use for the model. These parameters are all sent directly to the OpenAI + endpoint. See OpenAI [documentation](https://platform.openai.com/docs/api-reference/chat) for more details. + Some of the supported parameters: + - `max_tokens`: The maximum number of tokens the output text can have. + - `temperature`: What sampling temperature to use. Higher values mean the model will take more risks. + Try 0.9 for more creative applications, and 0 (argmax sampling) for ones with a well-defined answer. + - `top_p`: An alternative to sampling with temperature, called nucleus sampling, where the model + considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens + comprising the top 10% probability mass are considered. + - `n`: How many completions to generate for each prompt. For example, if the LLM gets 3 prompts and n is 2, + it will generate two completions for each of the three prompts, ending up with 6 completions in total. + - `stop`: One or more sequences after which the LLM should stop generating tokens. + - `presence_penalty`: What penalty to apply if a token is already present at all. Bigger values mean + the model will be less likely to repeat the same token in the text. + - `frequency_penalty`: What penalty to apply if a token has already been generated in the text. + Bigger values mean the model will be less likely to repeat the same token in the text. + - `logit_bias`: Add a logit bias to specific tokens. The keys of the dictionary are tokens and the + values are the bias to add to that token. + """ + super().__init__( + api_key=api_key, + model_name=model_name, + system_prompt=system_prompt, + streaming_callback=streaming_callback, + api_base_url=api_base_url, + **kwargs, + ) From 31ebfacd3c0f08a4c38a1700f28ec01aba5ff42c Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 10:10:30 +0200 Subject: [PATCH 2/6] add e2e --- .../components/test_gpt35_generator.py | 29 +++++++++++-------- .../components/generators/openai/gpt4.py | 3 +- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/e2e/preview/components/test_gpt35_generator.py b/e2e/preview/components/test_gpt35_generator.py index c70b8033f5..b7dde150de 100644 --- a/e2e/preview/components/test_gpt35_generator.py +++ b/e2e/preview/components/test_gpt35_generator.py @@ -2,14 +2,16 @@ import pytest import openai from haystack.preview.components.generators.openai.gpt35 import GPT35Generator +from haystack.preview.components.generators.openai.gpt4 import GPT4Generator @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY", None), reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", ) -def test_gpt35_generator_run(): - component = GPT35Generator(api_key=os.environ.get("OPENAI_API_KEY"), n=1) +@pytest.mark.parametrize("generator_class,model_name", [(GPT35Generator, "gpt-3.5"), (GPT4Generator, "gpt-4")]) +def test_gpt35_generator_run(generator_class, model_name): + component = generator_class(api_key=os.environ.get("OPENAI_API_KEY"), n=1) results = component.run(prompts=["What's the capital of France?", "What's the capital of Germany?"]) assert len(results["replies"]) == 2 @@ -20,10 +22,10 @@ def test_gpt35_generator_run(): assert len(results["metadata"]) == 2 assert len(results["metadata"][0]) == 1 - assert "gpt-3.5-turbo" in results["metadata"][0][0]["model"] + assert model_name in results["metadata"][0][0]["model"] assert "stop" == results["metadata"][0][0]["finish_reason"] assert len(results["metadata"][1]) == 1 - assert "gpt-3.5-turbo" in results["metadata"][1][0]["model"] + assert model_name in results["metadata"][1][0]["model"] assert "stop" == results["metadata"][1][0]["finish_reason"] @@ -31,8 +33,9 @@ def test_gpt35_generator_run(): not os.environ.get("OPENAI_API_KEY", None), reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", ) -def test_gpt35_generator_run_wrong_model_name(): - component = GPT35Generator(model_name="something-obviously-wrong", api_key=os.environ.get("OPENAI_API_KEY"), n=1) +@pytest.mark.parametrize("generator_class", [GPT35Generator, GPT4Generator]) +def test_gpt35_generator_run_wrong_model_name(generator_class): + component = generator_class(model_name="something-obviously-wrong", api_key=os.environ.get("OPENAI_API_KEY"), n=1) with pytest.raises(openai.InvalidRequestError, match="The model `something-obviously-wrong` does not exist"): component.run(prompts=["What's the capital of France?"]) @@ -41,8 +44,9 @@ def test_gpt35_generator_run_wrong_model_name(): not os.environ.get("OPENAI_API_KEY", None), reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", ) -def test_gpt35_generator_run_above_context_length(): - component = GPT35Generator(api_key=os.environ.get("OPENAI_API_KEY"), n=1) +@pytest.mark.parametrize("generator_class", [GPT35Generator, GPT4Generator]) +def test_gpt35_generator_run_above_context_length(generator_class): + component = generator_class(api_key=os.environ.get("OPENAI_API_KEY"), n=1) with pytest.raises( openai.InvalidRequestError, match="This model's maximum context length is 4097 tokens. However, your messages resulted in 70008 tokens. " @@ -55,7 +59,8 @@ def test_gpt35_generator_run_above_context_length(): not os.environ.get("OPENAI_API_KEY", None), reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.", ) -def test_gpt35_generator_run_streaming(): +@pytest.mark.parametrize("generator_class,model_name", [(GPT35Generator, "gpt-3.5"), (GPT4Generator, "gpt-4")]) +def test_gpt35_generator_run_streaming(generator_class, model_name): class Callback: def __init__(self): self.responses = "" @@ -65,7 +70,7 @@ def __call__(self, chunk): return chunk callback = Callback() - component = GPT35Generator(os.environ.get("OPENAI_API_KEY"), streaming_callback=callback, n=1) + component = generator_class(os.environ.get("OPENAI_API_KEY"), streaming_callback=callback, n=1) results = component.run(prompts=["What's the capital of France?", "What's the capital of Germany?"]) assert len(results["replies"]) == 2 @@ -79,8 +84,8 @@ def __call__(self, chunk): assert len(results["metadata"]) == 2 assert len(results["metadata"][0]) == 1 - assert "gpt-3.5-turbo" in results["metadata"][0][0]["model"] + assert model_name in results["metadata"][0][0]["model"] assert "stop" == results["metadata"][0][0]["finish_reason"] assert len(results["metadata"][1]) == 1 - assert "gpt-3.5-turbo" in results["metadata"][1][0]["model"] + assert model_name in results["metadata"][1][0]["model"] assert "stop" == results["metadata"][1][0]["finish_reason"] diff --git a/haystack/preview/components/generators/openai/gpt4.py b/haystack/preview/components/generators/openai/gpt4.py index f258f9c5d0..3ba5eb46e4 100644 --- a/haystack/preview/components/generators/openai/gpt4.py +++ b/haystack/preview/components/generators/openai/gpt4.py @@ -3,13 +3,14 @@ import logging from haystack.preview import component +from haystack.preview.components.generators.openai.gpt35 import GPT35Generator logger = logging.getLogger(__name__) @component -class GPT4Generator: +class GPT4Generator(GPT35Generator): """ LLM Generator compatible with GPT4 large language models. From c6f502915e256673a4c04dccd5e597f7914673ef Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 10:17:42 +0200 Subject: [PATCH 3/6] add tests --- .../components/generators/openai/gpt35.py | 5 +- .../components/generators/openai/gpt4.py | 4 +- .../generators/openai/test_gpt4_generator.py | 110 ++++++++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) create mode 100644 test/preview/components/generators/openai/test_gpt4_generator.py diff --git a/haystack/preview/components/generators/openai/gpt35.py b/haystack/preview/components/generators/openai/gpt35.py index e58b1ffa05..d0914c0b32 100644 --- a/haystack/preview/components/generators/openai/gpt35.py +++ b/haystack/preview/components/generators/openai/gpt35.py @@ -12,6 +12,9 @@ logger = logging.getLogger(__name__) +API_BASE_URL = "https://api.openai.com/v1" + + @dataclass class _ChatMessage: content: str @@ -43,7 +46,7 @@ def __init__( model_name: str = "gpt-3.5-turbo", system_prompt: Optional[str] = None, streaming_callback: Optional[Callable] = None, - api_base_url: str = "https://api.openai.com/v1", + api_base_url: str = API_BASE_URL, **kwargs, ): """ diff --git a/haystack/preview/components/generators/openai/gpt4.py b/haystack/preview/components/generators/openai/gpt4.py index 3ba5eb46e4..b50c96e9fe 100644 --- a/haystack/preview/components/generators/openai/gpt4.py +++ b/haystack/preview/components/generators/openai/gpt4.py @@ -3,7 +3,7 @@ import logging from haystack.preview import component -from haystack.preview.components.generators.openai.gpt35 import GPT35Generator +from haystack.preview.components.generators.openai.gpt35 import GPT35Generator, API_BASE_URL logger = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def __init__( model_name: str = "gpt-4", system_prompt: Optional[str] = None, streaming_callback: Optional[Callable] = None, - api_base_url: str = "https://api.openai.com/v1", + api_base_url: str = API_BASE_URL, **kwargs, ): """ diff --git a/test/preview/components/generators/openai/test_gpt4_generator.py b/test/preview/components/generators/openai/test_gpt4_generator.py new file mode 100644 index 0000000000..bb70af2acf --- /dev/null +++ b/test/preview/components/generators/openai/test_gpt4_generator.py @@ -0,0 +1,110 @@ +from unittest.mock import patch, Mock + +import pytest + +from haystack.preview.components.generators.openai.gpt4 import GPT4Generator, API_BASE_URL +from haystack.preview.components.generators.openai.gpt35 import default_streaming_callback + + +class TestGPT35Generator: + @pytest.mark.unit + def test_init_default(self): + component = GPT4Generator(api_key="test-api-key") + assert component.system_prompt is None + assert component.api_key == "test-api-key" + assert component.model_name == "gpt-4" + assert component.streaming_callback is None + assert component.api_base_url == API_BASE_URL + assert component.model_parameters == {} + + @pytest.mark.unit + def test_init_with_parameters(self): + callback = lambda x: x + component = GPT4Generator( + api_key="test-api-key", + model_name="gpt-4-32k", + system_prompt="test-system-prompt", + max_tokens=10, + some_test_param="test-params", + streaming_callback=callback, + api_base_url="test-base-url", + ) + assert component.system_prompt == "test-system-prompt" + assert component.api_key == "test-api-key" + assert component.model_name == "gpt-4-32k" + assert component.streaming_callback == callback + assert component.api_base_url == "test-base-url" + assert component.model_parameters == {"max_tokens": 10, "some_test_param": "test-params"} + + @pytest.mark.unit + def test_to_dict_default(self): + component = GPT4Generator(api_key="test-api-key") + data = component.to_dict() + assert data == { + "type": "GPT4Generator", + "init_parameters": { + "api_key": "test-api-key", + "model_name": "gpt-4", + "system_prompt": None, + "streaming_callback": None, + "api_base_url": API_BASE_URL, + }, + } + + @pytest.mark.unit + def test_to_dict_with_parameters(self): + component = GPT4Generator( + api_key="test-api-key", + model_name="gpt-4-32k", + system_prompt="test-system-prompt", + max_tokens=10, + some_test_param="test-params", + streaming_callback=default_streaming_callback, + api_base_url="test-base-url", + ) + data = component.to_dict() + assert data == { + "type": "GPT4Generator", + "init_parameters": { + "api_key": "test-api-key", + "model_name": "gpt-4-32k", + "system_prompt": "test-system-prompt", + "max_tokens": 10, + "some_test_param": "test-params", + "api_base_url": "test-base-url", + "streaming_callback": "haystack.preview.components.generators.openai.gpt35.default_streaming_callback", + }, + } + + @pytest.mark.unit + def test_from_dict_default(self): + data = {"type": "GPT4Generator", "init_parameters": {"api_key": "test-api-key"}} + component = GPT4Generator.from_dict(data) + assert component.system_prompt is None + assert component.api_key == "test-api-key" + assert component.model_name == "gpt-4" + assert component.streaming_callback is None + assert component.api_base_url == API_BASE_URL + assert component.model_parameters == {} + + @pytest.mark.unit + def test_from_dict(self): + data = { + "type": "GPT4Generator", + "init_parameters": { + "api_key": "test-api-key", + "model_name": "gpt-4-32k", + "system_prompt": "test-system-prompt", + "max_tokens": 10, + "some_test_param": "test-params", + "api_base_url": "test-base-url", + "streaming_callback": "haystack.preview.components.generators.openai.gpt35.default_streaming_callback", + }, + } + component = GPT4Generator.from_dict(data) + assert component.system_prompt == "test-system-prompt" + assert component.api_key == "test-api-key" + assert component.model_name == "gpt-4-32k" + assert component.streaming_callback == default_streaming_callback + assert component.api_base_url == "test-base-url" + assert component.model_parameters == {"max_tokens": 10, "some_test_param": "test-params"} From a7e0f7a16cf313b75b99f7e813f7e07176e0554f Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 10:21:16 +0200 Subject: [PATCH 4/6] reno --- releasenotes/notes/gpt4-llm-generator-60708087ec42211f.yaml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 releasenotes/notes/gpt4-llm-generator-60708087ec42211f.yaml diff --git a/releasenotes/notes/gpt4-llm-generator-60708087ec42211f.yaml b/releasenotes/notes/gpt4-llm-generator-60708087ec42211f.yaml new file mode 100644 index 0000000000..fb67de3ccd --- /dev/null +++ b/releasenotes/notes/gpt4-llm-generator-60708087ec42211f.yaml @@ -0,0 +1,3 @@ + +preview: + - Adds `GPT4Generator`, an LLM component based on `GPT35Generator` From f7d15a3e12a5b897f43e45a821f4cbf392f43a22 Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 16:25:07 +0200 Subject: [PATCH 5/6] fix e2e --- e2e/preview/components/test_gpt35_generator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/e2e/preview/components/test_gpt35_generator.py b/e2e/preview/components/test_gpt35_generator.py index b7dde150de..725ce3d44e 100644 --- a/e2e/preview/components/test_gpt35_generator.py +++ b/e2e/preview/components/test_gpt35_generator.py @@ -49,10 +49,9 @@ def test_gpt35_generator_run_above_context_length(generator_class): component = generator_class(api_key=os.environ.get("OPENAI_API_KEY"), n=1) with pytest.raises( openai.InvalidRequestError, - match="This model's maximum context length is 4097 tokens. However, your messages resulted in 70008 tokens. " - "Please reduce the length of the messages.", + match="However, your messages resulted in 35008 tokens. Please reduce the length of the messages.", ): - component.run(prompts=["What's the capital of France? " * 10_000]) + component.run(prompts=["What's the capital of France? " * 5_000]) @pytest.mark.skipif( From 2068a42dd3aa90d68a777e255e2fb1b988b43dde Mon Sep 17 00:00:00 2001 From: ZanSara Date: Fri, 8 Sep 2023 16:25:40 +0200 Subject: [PATCH 6/6] Update test/preview/components/generators/openai/test_gpt4_generator.py Co-authored-by: Stefano Fiorucci <44616784+anakin87@users.noreply.github.com> --- .../preview/components/generators/openai/test_gpt4_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/preview/components/generators/openai/test_gpt4_generator.py b/test/preview/components/generators/openai/test_gpt4_generator.py index bb70af2acf..8c53b40c8e 100644 --- a/test/preview/components/generators/openai/test_gpt4_generator.py +++ b/test/preview/components/generators/openai/test_gpt4_generator.py @@ -6,7 +6,7 @@ from haystack.preview.components.generators.openai.gpt35 import default_streaming_callback -class TestGPT35Generator: +class TestGPT4Generator: @pytest.mark.unit def test_init_default(self): component = GPT4Generator(api_key="test-api-key")