diff --git a/example/app/lib/ollama.rb b/example/app/lib/ollama.rb new file mode 100644 index 00000000..78131e1f --- /dev/null +++ b/example/app/lib/ollama.rb @@ -0,0 +1,63 @@ +class Ollama + DEFAULT_URL = "http://localhost:11434" + DEFAULT_MODEL = "llama2" + + GenerateResponse = + Data.define( + :total_duration, # time spent generating the response + :load_duration, # time spent in nanoseconds loading the model + :sample_count, # number of samples generated + :sample_duration, # time spent generating samples + :prompt_eval_count, # number of tokens in the prompt + :prompt_eval_duration, # time spent in nanoseconds evaluating the prompt + :eval_count, # number of tokens the response + :eval_duration, # time in nanoseconds spent generating the response + :model, + :created_at, + :context # an encoding of the conversation used in this response, this can be sent in the next request to keep a conversational memory + ) + + def initialize(model: DEFAULT_MODEL, url: DEFAULT_URL) + @endpoint = + Async::HTTP::Endpoint.parse(url, protocol: Async::HTTP::Protocol::HTTP2) + @client = Async::HTTP::Client.new(@endpoint) + @model = model + @context = nil + end + + def generate(prompt, system: nil, template: nil, options: {}) + res = + @client.post( + "/api/generate", + nil, + JSON.generate( + { + model: @model, + prompt:, + context: @context, + template:, + options:, + system: + } + ) + ) + + chunks = [] + + res.each do |chunk| + parsed = JSON.parse(chunk, symbolize_names: true) + + case parsed + in error: + raise error.to_s + in { done: true, context: } + @context = context + in response: + chunks << response + yield response + end + end + + chunks.join.strip + end +end diff --git a/example/app/pages/demos/layout.haml b/example/app/pages/demos/layout.haml index f8d13393..7934b1c0 100644 --- a/example/app/pages/demos/layout.haml +++ b/example/app/pages/demos/layout.haml @@ -7,6 +7,7 @@ LINKS = { "/demos" => "Demos", "/demos/pokemon" => "Pokémon", + "/demos/ollama" => "Ollama chat", "/demos/tree" => "App tree", "/demos/form" => "Form elements", "/demos/images" => "Images", diff --git a/example/app/pages/demos/ollama/Message.haml b/example/app/pages/demos/ollama/Message.haml new file mode 100644 index 00000000..e1729904 --- /dev/null +++ b/example/app/pages/demos/ollama/Message.haml @@ -0,0 +1,24 @@ +%li{class: $role.to_sym} + %strong= $role + %span= $text + +:css + li { + display: block; + padding: .5em; + margin: 0; + } + + strong { + &::after { + content: ": "; + } + } + + .user { + background: #0063; + } + + .model { + background: #0603; + } diff --git a/example/app/pages/demos/ollama/page.haml b/example/app/pages/demos/ollama/page.haml new file mode 100644 index 00000000..3caad2e9 --- /dev/null +++ b/example/app/pages/demos/ollama/page.haml @@ -0,0 +1,141 @@ +:ruby + Heading = import("/app/components/Layout/Heading") + Button = import("/app/components/Form/Button") + Message = import("./Message") + + MODEL = "llama2" + + def self.get_initial_state(**) = { + key: 0, + messages: [], + words: [], + loading: false, + ollama: Ollama.new(model: MODEL, url: ENV["OLLAMA_URL"]) + } + + def handle_submit(e) + return if state[:loading] + + text = e.dig(:currentTarget, :formData, :message) + + update do |state| + { + **state, + key: state[:key].succ, + loading: true, + messages: [ + *state[:messages], + { id: SecureRandom.alphanumeric, role: "user", text: } + ] + } + end + + chunks = [] + + begin + state[:ollama].generate(text) do |word| + chunks.push(word) + + update do |state| + { + **state, + words: [*state[:words], word] + } + end + end + rescue => e + pp e + end + + update do |state| + { + **state, + words: [], + messages: [ + *state[:messages], + { + id: SecureRandom.alphanumeric, + role: "model", + text: chunks.join.strip + } + ] + } + end + ensure + update(loading: false) + end + +%section + %Heading(level=2) Ollama chat + + .scroller + = if state[:messages].empty? + %p.type-your-message Type a message to chat with #{MODEL} + %ul + = unless state[:words].empty? + %Message.model[:temp]{ + role: "model", + text: state[:words].join.strip + } + = state[:messages].reverse.map do |message| + %Message.model[message[:id]]{ + role: message[:role], + text: message[:text], + } + %form(onsubmit=handle_submit) + %input[state[:key]]{ + autofocus: true, + type: "text", + name: "message", + autocomplete: "off", + placeholder: "Type your message here…" + } + %Button(type="submit"){disabled: state[:loading]} Send + +:css + section { + display: grid; + grid-template-rows: auto 1fr auto; + min-height: 20em; + gap: 1em; + } + + Heading { + margin-bottom: 0; + } + + .scroller { + position: relative; + } + + ul { + position: absolute; + inset: 0; + overflow-y: scroll; + font-family: "Roboto Mono"; + white-space: pre-wrap; + border: 1px solid #0003; + border-radius: 3px; + display: flex; + flex-direction: column-reverse; + margin: 0; + padding: 0; + } + + form { + display: grid; + grid-template-columns: 1fr auto; + gap: 1em; + } + + input { + padding: .5em; + } + + .type-your-message { + position: absolute; + top: 50%; + left: 50%; + transform: translate(-50%, -50%); + font-weight: bold; + } diff --git a/example/bin/mayu b/example/bin/mayu index b087f3a4..84aff037 100755 --- a/example/bin/mayu +++ b/example/bin/mayu @@ -4,4 +4,6 @@ require "rubygems" require "bundler/setup" +require_relative "../app/lib/ollama" + load Gem.bin_path("mayu-live", "mayu")