diff --git a/examples/applications/ollama-chatbot/.gitignore b/examples/applications/ollama-chatbot/.gitignore new file mode 100644 index 000000000..55dea2dd3 --- /dev/null +++ b/examples/applications/ollama-chatbot/.gitignore @@ -0,0 +1 @@ +java/lib/* \ No newline at end of file diff --git a/examples/applications/ollama-chatbot/README.md b/examples/applications/ollama-chatbot/README.md new file mode 100644 index 000000000..8d321c573 --- /dev/null +++ b/examples/applications/ollama-chatbot/README.md @@ -0,0 +1,96 @@ +# Running your own Chat bot using docker + +This sample application shows how to build a chat bot over the content of a website. +In this case you are going to crawl the LangStream.ai documentation website. + +The Chat bot will be able to help you with LangStream. + +In this example we are using [HerdDB](ps://github.com/diennea/herddb) as a vector database using the JDBC driver, +but you can use any Vector databases. + + +## Configure you OpenAI API Key + +Export to the ENV the access key to OpenAI + +``` +export OPEN_AI_ACCESS_KEY=... +``` + +The default [secrets file](../../secrets/secrets.yaml) reads from the ENV. Check out the file to learn more about +the default settings, you can change them by exporting other ENV variables. + +## Deploy the LangStream application in docker + +The default docker runner starts Minio, Kafka and HerdDB, so you can run the application locally. + +``` +./bin/langstream docker run test -app examples/applications/docker-chatbot -s examples/secrets/secrets.yaml +``` + + +## Talk with the Chat bot using the CLI +Since the application opens a gateway, we can use the gateway API to send and consume messages. + +``` +./bin/langstream gateway chat test -cg bot-output -pg user-input -p sessionId=$(uuidgen) +``` + +Responses are streamed to the output-topic. If you want to inspect the history of the raw answers you can +consume from the log-topic using the llm-debug gateway: + +``` +./bin/langstream gateway consume test llm-debug +``` + +## Application flow chart + +```mermaid +flowchart TB + + subgraph JdbcDatasource["⛁ JdbcDatasource"] + documents + end + + subgraph streaming-cluster["✉️ streaming cluster"] + questions-topic + answers-topic + log-topic + chunks-topic + end + + subgraph gateways["gateways"] + user-input --> questions-topic + answers-topic --> bot-output + log-topic --> llm-debug + end + + subgraph chatbot["chatbot"] + A("convert-to-structure
document-to-json") --> B + B("compute-embeddings
compute-ai-embeddings") --> C + C("lookup-related-documents
query-vector-db") --> D + D("re-rank documents with MMR
re-rank") --> E + E("ai-chat-completions
ai-chat-completions") --> F + F("cleanup-response
drop-fields") + end + questions-topic --> A + JdbcDatasource --> C + E --> answers-topic + F --> log-topic + + subgraph crawler["crawler"] + G("Crawl the WebSite
webcrawler-source") --> H + H("Extract text
text-extractor") --> I + I("Normalise text
text-normaliser") --> J + J("Detect language
language-detector") --> K + K("Split into chunks
text-splitter") --> L + L("Convert to structured data
document-to-json") --> M + M("prepare-structure
compute") --> N + N("compute-embeddings
compute-ai-embeddings") + O("Write
vector-db-sink") + end + P["🌐 web site"] --> G + N --> chunks-topic + chunks-topic --> O + O --> documents +``` diff --git a/examples/applications/ollama-chatbot/assets.yaml b/examples/applications/ollama-chatbot/assets.yaml new file mode 100644 index 000000000..bedbdaf0f --- /dev/null +++ b/examples/applications/ollama-chatbot/assets.yaml @@ -0,0 +1,32 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +assets: + - name: "documents-table" + asset-type: "jdbc-table" + creation-mode: create-if-not-exists + config: + table-name: "documents" + datasource: "JdbcDatasource" + create-statements: + - | + CREATE TABLE documents ( + filename TEXT, + chunk_id int, + num_tokens int, + lang TEXT, + text TEXT, + embeddings_vector FLOATA, + PRIMARY KEY (filename, chunk_id)); \ No newline at end of file diff --git a/examples/applications/ollama-chatbot/chatbot.yaml b/examples/applications/ollama-chatbot/chatbot.yaml new file mode 100644 index 000000000..6b90c156d --- /dev/null +++ b/examples/applications/ollama-chatbot/chatbot.yaml @@ -0,0 +1,109 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +topics: + - name: "questions-topic" + creation-mode: create-if-not-exists + - name: "answers-topic" + creation-mode: create-if-not-exists + - name: "log-topic" + creation-mode: create-if-not-exists +errors: + on-failure: "skip" +pipeline: + - name: "convert-to-structure" + type: "document-to-json" + input: "questions-topic" + configuration: + text-field: "question" + - name: "compute-embeddings" + type: "compute-ai-embeddings" + resources: + disk: + size: 256M + enabled: true + configuration: + model: "multilingual-e5-small" + model-url: "djl://ai.djl.huggingface.pytorch/intfloat/multilingual-e5-small" + ai-service: "huggingface" + embeddings-field: "value.question_embeddings" + text: "{{ value.question }}" + flush-interval: 0 + - name: "lookup-related-documents" + type: "query-vector-db" + configuration: + datasource: "JdbcDatasource" + query: "SELECT text,embeddings_vector FROM documents ORDER BY cosine_similarity(embeddings_vector, CAST(? as FLOAT ARRAY)) DESC LIMIT 20" + fields: + - "value.question_embeddings" + output-field: "value.related_documents" + - name: "re-rank documents with MMR" + type: "re-rank" + configuration: + max: 5 # keep only the top 5 documents, because we have an hard limit on the prompt size + field: "value.related_documents" + query-text: "value.question" + query-embeddings: "value.question_embeddings" + output-field: "value.related_documents" + text-field: "record.text" + embeddings-field: "record.embeddings_vector" + algorithm: "MMR" + lambda: 0.5 + k1: 1.2 + b: 0.75 + - name: "ai-chat-completions" + type: "ai-chat-completions" + + configuration: + ai-service: "ollama" + model: "${secrets.ollama.model}" + # on the log-topic we add a field with the answer + completion-field: "value.answer" + # we are also logging the prompt we sent to the LLM + log-field: "value.prompt" + # here we configure the streaming behavior + # as soon as the LLM answers with a chunk we send it to the answers-topic + stream-to-topic: "answers-topic" + # on the streaming answer we send the answer as whole message + # the 'value' syntax is used to refer to the whole value of the message + stream-response-completion-field: "value" + # we want to stream the answer as soon as we have 20 chunks + # in order to reduce latency for the first message the agent sends the first message + # with 1 chunk, then with 2 chunks....up to the min-chunks-per-message value + # eventually we want to send bigger messages to reduce the overhead of each message on the topic + min-chunks-per-message: 20 + messages: + - role: system + content: | + An user is going to perform a questions, The documents below may help you in answering to their questions. + Please try to leverage them in your answer as much as possible. + Take into consideration that the user is always asking questions about the LangStream project. + If you provide code or YAML snippets, please explicitly state that they are examples. + Do not provide information that is not related to the LangStream project. + + Documents: + {{# value.related_documents}} + {{ text}} + {{/ value.related_documents}} + - role: user + content: "{{ value.question}}" + - name: "cleanup-response" + type: "drop-fields" + output: "log-topic" + configuration: + fields: + - "question_embeddings" + - "related_documents" \ No newline at end of file diff --git a/examples/applications/ollama-chatbot/configuration.yaml b/examples/applications/ollama-chatbot/configuration.yaml new file mode 100644 index 000000000..34d167d58 --- /dev/null +++ b/examples/applications/ollama-chatbot/configuration.yaml @@ -0,0 +1,40 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +configuration: + resources: + - type: "datasource" + name: "JdbcDatasource" + configuration: + service: "jdbc" + driverClass: "herddb.jdbc.Driver" + url: "${secrets.herddb.url}" + user: "${secrets.herddb.user}" + password: "${secrets.herddb.password}" + - type: "ollama-configuration" + name: "ollama" + configuration: + url: "${secrets.ollama.url}" + - type: "hugging-face-configuration" + name: "huggingface" + configuration: + provider: "local" + dependencies: + - name: "HerdDB.org JDBC Driver" + url: "https://repo1.maven.org/maven2/org/herddb/herddb-jdbc/0.28.0/herddb-jdbc-0.28.0-thin.jar" + sha512sum: "d8ea8fbb12eada8f860ed660cbc63d66659ab3506bc165c85c420889aa8a1dac53dab7906ef61c4415a038c5a034f0d75900543dd0013bdae50feafd46f51c8e" + type: "java-library" \ No newline at end of file diff --git a/examples/applications/ollama-chatbot/crawler.yaml b/examples/applications/ollama-chatbot/crawler.yaml new file mode 100644 index 000000000..3ed22b8b2 --- /dev/null +++ b/examples/applications/ollama-chatbot/crawler.yaml @@ -0,0 +1,121 @@ +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +name: "Crawl a website" +resources: + size: 2 +pipeline: + - name: "Crawl the WebSite" + type: "webcrawler-source" + configuration: + seed-urls: ["https://docs.langstream.ai/"] + allowed-domains: ["https://docs.langstream.ai"] + forbidden-paths: [] + min-time-between-requests: 500 + max-error-count: 5 + max-urls: 1000 + max-depth: 50 + handle-robots-file: true + scan-html-documents: true + state-storage: disk + - name: "Extract text" + type: "text-extractor" + - name: "Normalise text" + type: "text-normaliser" + configuration: + make-lowercase: true + trim-spaces: true + - name: "Detect language" + type: "language-detector" + configuration: + allowedLanguages: ["en", "fr"] + property: "language" + - name: "Split into chunks" + type: "text-splitter" + configuration: + splitter_type: "RecursiveCharacterTextSplitter" + chunk_size: 400 + separators: ["\n\n", "\n", " ", ""] + keep_separator: false + chunk_overlap: 100 + length_function: "cl100k_base" + - name: "Convert to structured data" + type: "document-to-json" + configuration: + text-field: text + copy-properties: true + - name: "prepare-structure" + type: "compute" + configuration: + fields: + - name: "value.filename" + expression: "properties.url" + type: STRING + - name: "value.chunk_id" + expression: "properties.chunk_id" + type: STRING + - name: "value.language" + expression: "properties.language" + type: STRING + - name: "value.chunk_num_tokens" + expression: "properties.chunk_num_tokens" + type: STRING + - name: "compute-embeddings" + id: "step1" + type: "compute-ai-embeddings" + resources: + disk: + size: 256M + enabled: true + configuration: + model: "multilingual-e5-small" + model-url: "djl://ai.djl.huggingface.pytorch/intfloat/multilingual-e5-small" + ai-service: "huggingface" + embeddings-field: "value.embeddings_vector" + text: "{{ value.text }}" + batch-size: 10 + flush-interval: 500 + - name: "Delete stale chunks" + type: "query" + configuration: + datasource: "JdbcDatasource" + when: "fn:toInt(properties.text_num_chunks) == (fn:toInt(properties.chunk_id) + 1)" + mode: "execute" + query: "DELETE FROM documents WHERE filename = ? AND chunk_id > ?" + output-field: "value.delete-results" + fields: + - "value.filename" + - "fn:toInt(value.chunk_id)" + - name: "Write" + type: "vector-db-sink" + configuration: + datasource: "JdbcDatasource" + table-name: "documents" + fields: + - name: "filename" + expression: "value.filename" + primary-key: true + - name: "chunk_id" + expression: "value.chunk_id" + primary-key: true + - name: "embeddings_vector" + expression: "fn:toListOfFloat(value.embeddings_vector)" + - name: "lang" + expression: "value.language" + - name: "text" + expression: "value.text" + - name: "num_tokens" + expression: "value.chunk_num_tokens" \ No newline at end of file diff --git a/examples/applications/ollama-chatbot/gateways.yaml b/examples/applications/ollama-chatbot/gateways.yaml new file mode 100644 index 000000000..132788270 --- /dev/null +++ b/examples/applications/ollama-chatbot/gateways.yaml @@ -0,0 +1,43 @@ +# +# +# Copyright DataStax, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +gateways: + - id: "user-input" + type: produce + topic: "questions-topic" + parameters: + - sessionId + produceOptions: + headers: + - key: langstream-client-session-id + valueFromParameters: sessionId + + - id: "bot-output" + type: consume + topic: "answers-topic" + parameters: + - sessionId + consumeOptions: + filters: + headers: + - key: langstream-client-session-id + valueFromParameters: sessionId + + + - id: "llm-debug" + type: consume + topic: "log-topic" \ No newline at end of file diff --git a/examples/secrets/secrets.yaml b/examples/secrets/secrets.yaml index f93be05eb..51123197f 100644 --- a/examples/secrets/secrets.yaml +++ b/examples/secrets/secrets.yaml @@ -144,6 +144,11 @@ secrets: secret-key: "${BEDROCK_SECRET_KEY}" region: "${REGION:-us-east-1}" completions-model: "${BEDROCK_COMPLETIONS_MODEL}" + - name: ollama + id: ollama + data: + url: "${OLLAMA_URL:-http://host.docker.internal:11434/api/generate}" + model: "${OLLAMA_MODEL:-llama2}" - name: camel-github-source id: camel-github-source data: diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/OllamaConfig.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/OllamaConfig.java new file mode 100644 index 000000000..d180e7b5a --- /dev/null +++ b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/OllamaConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.streaming.ai.model.config; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class OllamaConfig { + + @JsonProperty(value = "url") + private String url; +} diff --git a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/TransformStepConfig.java b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/TransformStepConfig.java index 13392ead5..88e3befbd 100644 --- a/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/TransformStepConfig.java +++ b/langstream-agents/langstream-ai-agents/src/main/java/com/datastax/oss/streaming/ai/model/config/TransformStepConfig.java @@ -31,6 +31,8 @@ public class TransformStepConfig { @JsonProperty private BedrockConfig bedrock; + @JsonProperty private OllamaConfig ollama; + @JsonProperty private Map datasource; @JsonProperty private boolean attemptJsonConversion = true; diff --git a/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/services/impl/OllamaProviderTest.java b/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/services/impl/OllamaProviderTest.java index 474a0f337..230e7f3e1 100644 --- a/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/services/impl/OllamaProviderTest.java +++ b/langstream-agents/langstream-ai-agents/src/test/java/ai/langstream/ai/agents/services/impl/OllamaProviderTest.java @@ -24,9 +24,17 @@ import com.github.tomakehurst.wiremock.junit5.WireMockTest; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.Test; +import static com.github.tomakehurst.wiremock.client.WireMock.ok; +import static com.github.tomakehurst.wiremock.client.WireMock.okJson; +import static com.github.tomakehurst.wiremock.client.WireMock.post; +import static com.github.tomakehurst.wiremock.client.WireMock.stubFor; +import static org.junit.jupiter.api.Assertions.assertEquals; + @Slf4j @WireMockTest class OllamaProviderTest { @@ -34,12 +42,22 @@ class OllamaProviderTest { @Test void testCompletions(WireMockRuntimeInfo wmRuntimeInfo) throws Exception { + stubFor( + post("/api/generate") + .willReturn( + ok( + """ + {"model":"llama2","created_at":"2023-11-09T13:48:51.788062Z","response":"one","done":false} + {"model":"llama2","created_at":"2023-11-09T13:48:51.788062Z","response":" two","done":false} + {"model":"llama2","created_at":"2023-11-09T13:48:51.788062Z","response":" three","done":true} + """))); OllamaProvider provider = new OllamaProvider(); ServiceProvider implementation = provider.createImplementation( - Map.of("ollama", Map.of("url", "http://localhost:11434/api/generate")), + Map.of("ollama", Map.of("url", wmRuntimeInfo.getHttpBaseUrl() + "/api/generate")), MetricsReporter.DISABLED); + List chunks = new CopyOnWriteArrayList<>(); CompletionsService service = implementation.getCompletionsService(Map.of("model", "llama2")); String result = @@ -47,7 +65,7 @@ void testCompletions(WireMockRuntimeInfo wmRuntimeInfo) throws Exception { List.of( new ChatMessage("user") .setContent( - "Here is a story about llamas eating grass")), + "Tell me three numberss")), new CompletionsService.StreamingChunksConsumer() { @Override public void consumeChunk( @@ -58,6 +76,7 @@ public void consumeChunk( index, chunk, last); + chunks.add(chunk.content()); } }, Map.of()) @@ -66,5 +85,7 @@ public void consumeChunk( .get(0) .content(); log.info("result: {}", result); + assertEquals("one two three", result); + assertEquals(List.of("one", " two", " three"), chunks); } }