diff --git a/docusaurus.config.js b/docusaurus.config.js index 7d752fed..ed99d1e0 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -45,6 +45,17 @@ const config = { /** @type {import('@docusaurus/preset-classic').Options} */ ({ docs: { + lastVersion: "current", + versions: { + current: { + label: "1.0", + badge: false, + }, + "2.0": { + label: "2.0", + banner: "unreleased", + }, + }, sidebarPath: require.resolve("./sidebars.js"), remarkPlugins: [ [require("@docusaurus/remark-plugin-npm2yarn"), { sync: true }], diff --git a/versioned_docs/version-2.0/concepts.md b/versioned_docs/version-2.0/concepts.md new file mode 100644 index 00000000..c3a990d5 --- /dev/null +++ b/versioned_docs/version-2.0/concepts.md @@ -0,0 +1,32 @@ +# Concepts + +Explanations, clarification and discussion of key topics in LangSmith. + +### Admin + +- [ ] Organizations +- [ ] Workspaces +- [ ] Users +- [ ] API Keys +- [ ] Personal Access Tokens +- [ ] Roles + +### Tracing + +- [ ] Projects +- [ ] Traces +- [ ] Runs +- [ ] Tags +- [ ] Metadata + +### Evaluation + +- [ ] Prompts +- [ ] Experiment +- [ ] Datasets +- [ ] Feedback +- [ ] Evaluator + +- [ ] Annotation Queue +- [ ] Prompt +- [ ] Playground diff --git a/versioned_docs/version-2.0/how_to_guides.md b/versioned_docs/version-2.0/how_to_guides.md new file mode 100644 index 00000000..8b65b9a3 --- /dev/null +++ b/versioned_docs/version-2.0/how_to_guides.md @@ -0,0 +1,105 @@ +# How-To Guides + +Step-by-step guides that cover key tasks, operations and common problems. + +### Setup + +- [ ] Create an account and API Key +- [ ] Setup an organization +- [ ] Setup a workspace +- [ ] Setup Billing +- [ ] Assigning Roles (enterprise only) + +### Deployment + +- [ ] Self-hosting LangSmith on Kubernetes +- [ ] Self-hosting LangSmith via Docker Compose +- [ ] Setting up Auth + +### Tracing + +- [ ] Annotate code for tracing + - [ ] `traceable` + - [ ] wrapping OpenAI or Custom LLM + - [ ] RunTree +- [ ] Log traces to specific project +- [ ] Toggle tracing on and off +- [ ] Sample traces +- [ ] Add metadata and tags to traces +- [ ] Get run_id and trace_id +- [ ] Mask inputs and outputs +- [ ] Log a trace using LangChain +- [ ] Log a trace using instructor +- [ ] Exporting traces + - [ ] Link to data format in reference section +- [ ] Log multi-modal traces + +### Datasets + +- [ ] Create a dataset in the application +- [ ] Create a dataset using the API +- [ ] Export datasets +- [ ] Import datasets +- [ ] Version datasets +- [ ] Add metadata to examples +- [ ] Filter examples from a dataset +- [ ] Add a trace to a dataset + +### Evaluation + +- [ ] Run an evaluation +- [ ] Run an evaluation from the playground +- [ ] Run an evaluation on a particular version of dataset +- [ ] Run an evaluation on subset of dataset +- [ ] Use off-the-shelf LangChain evaluators +- [ ] Use custom evaluators +- [ ] Evaluate on intermediate steps +- [ ] Compare experiment results +- [ ] Track regressions and improvements +- [ ] Export experiment +- [ ] Unit test LLM applications +- [ ] View trace for an evaluation run +- [ ] Run a pairwise evaluation (coming soon) +- [ ] Audit evaluation scores (coming soon) + +### Human Feedback + +- [ ] Attach user feedback from your application to traces +- [ ] Annotate traces inline +- [ ] Add trace to annotation queue +- [ ] Annotate traces in the annotation queue + +### Monitoring and Automations + +- [ ] Filter for runs +- [ ] Use a trace filter +- [ ] View the monitor charts +- [ ] Slice chart by metadata and tag +- [ ] Set up a rule + - [ ] Online evaluation + - [ ] Annotation Queue addition + - [ ] Dataset addition + - [ ] Webhook action +- [ ] Group traces as threads +- [ ] View threads + +### Prompt Hub + +- [ ] Create a prompt +- [ ] Update a prompt +- [ ] Pull prompts in code +- [ ] Open a prompt from a trace +- [ ] Open a prompt from an experiment + +### Playground + +- [ ] Run a prompt in the playground +- [ ] Run a prompt on a custom model + +### Proxy + +- [ ] Run proxy +- [ ] Make a request to the proxy +- [ ] Turn off caching +- [ ] Stream results +- [ ] Turn on tracing diff --git a/versioned_docs/version-2.0/index.mdx b/versioned_docs/version-2.0/index.mdx new file mode 100644 index 00000000..e7940dca --- /dev/null +++ b/versioned_docs/version-2.0/index.mdx @@ -0,0 +1,177 @@ +--- +sidebar_label: Quick Start +sidebar_position: 1 +table_of_contents: true +--- + +import Tabs from "@theme/Tabs"; +import CodeBlock from "@theme/CodeBlock"; +import { + CodeTabs, + PythonBlock, + TypeScriptBlock, +} from "@site/src/components/InstructionsWithCode"; +import { + LangChainInstallationCodeTabs, + LangChainQuickStartCodeTabs, + ConfigureEnvironmentCodeTabs, + RunTreeQuickStartCodeTabs, + ConfigureSDKEnvironmentCodeTabs, + PythonSDKTracingCode, + TypeScriptSDKTracingCode, +} from "@site/src/components/QuickStart"; +import { ClientInstallationCodeTabs } from "@site/src/components/ClientInstallation"; +import DocCardList from "@theme/DocCardList"; + +# Getting started with LangSmith (New!) + +## Introduction + +**LangSmith** is a platform for building production-grade LLM applications. It allows you to closely monitor and evaluate your application, so you can ship quickly and with confidence. Use of LangChain is not necessary - LangSmith works on its own! + +## Install LangSmith + +We offer Python and Typescript SDKs for all your LangSmith needs. + + + +## Create an API key + +To create an API key head to the [setting pages](https://smith.langchain.com/settings). Then click **Create API Key.** + +## Setup your environment + + + +## Log your first trace + +

+ We provide multiple ways to log traces to LangSmith. Below, we'll highlight + how to use traceable. See more on the{" "} + Integrations page. +

+ + +- View a [sample output trace](https://smith.langchain.com/public/b37ca9b1-60cd-4a2a-817e-3c4e4443fdc0/r). +- Learn more about tracing on the [tracing page](/tracing). + +## Create your first evaluation + +Evalution requires a system to test, [data](evaluation/faq) to serve as test cases, and optionally evaluators to grade the results. Here we use a built-in accuracy evaluator. + + => { + return { + key: 'exact_match', + score: run.outputs?.output === example?.outputs?.output ? 1 : 0, + }; +};\n +await runOnDataset( + (input: { postfix: string }) => ({ output: \`Welcome $\{input.postfix\}\` }), // Your AI system goes here + datasetName, // The data to predict and grade over + { + evaluationConfig: { customEvaluators: [exactMatch] }, + projectMetadata: { + version: "1.0.0", + revision_id: "beta", + }, + } +);`, + }, + ]} + groupId="client-language" +/> diff --git a/versioned_docs/version-2.0/pricing.mdx b/versioned_docs/version-2.0/pricing.mdx new file mode 100644 index 00000000..32998117 --- /dev/null +++ b/versioned_docs/version-2.0/pricing.mdx @@ -0,0 +1,150 @@ +--- +sidebar_label: Pricing +sidebar_position: 4 +--- + +# Pricing + +## Plans + +| Startups | Developer | Plus | Enterprise | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Designed for early stage startups building AI applications | Designed for hobbyists who want to start their adventure solo | Everything in Developer, plus team features, higher rate limits, and longer data retention | Designed for teams with more security, deployment, and support needs | +| [Contact us to learn more](https://airtable.com/app8ZrGLtHAtFVO1o/pagfLAmdTz4ep7TGu/form) |
  • Free for 1 user
  • 5,000 free traces per month
  • Additional traces billed @ $0.005/trace
|
  • $39/user
  • 10,000 free traces per month
  • Additional traces billed @ $0.005/trace
| Custom | +| What to expect:

We want all early stage companies to build with LangSmith. LangSmith for Startups offers discounted prices and a generous free, monthly trace allotment, so you can have the right tooling in place as you grow your business. | Key features:
  • 1 Developer seat
  • Debugging traces
  • Dataset collection
  • Testing and evaluation
  • Prompt management
  • Monitoring

Education Users:
  • Sign up with an active .edu address, and get access to even more free monthly traces!
| Key features:
  • All features in Developer tier
  • Up to 10 seats
  • Hosted LangServe (beta)
  • Longer data retention
  • Higher rate limits
  • Email support
| Key features:
  • All features in Plus tier
  • Single Sign On (SSO)
  • Negotiable SLAs
  • Deployment options in customer’s environment
  • Custom rate limits
  • Team trainings
  • Shared Slack channel
  • Architectural guidance
  • Dedicated customer success manager
| + +## Plan Comparison + +| | Developer | Plus | Enterprise | +| ------------------------------------------- | :------------------------------------------------------------------------------: | :-------------------------------------------------------------------------------: | :-------------------------------------------------------: | +| Features | | | | +| Debugging Traces | ✅ | ✅ | ✅ | +| Dataset Collection | ✅ | ✅ | ✅ | +| Human Labeling | ✅ | ✅ | ✅ | +| Testing and Evaluation | ✅ | ✅ | ✅ | +| Prompt Management | ✅ | ✅ | ✅ | +| Hosted LangServe | -- | ✅ | ✅ | +| Monitoring | ✅ | ✅ | ✅ | +| Team | | | | +| Developer Seats | 1 Free Seat | Maximum 10 seats
$39 per seat/month1 | Unlimited seats
Custom pricing | +| Collaborator Seats | -- | -- | Coming Soon! | +| Trace Details | | | | +| Traces | First 5k traces per month for free.
$0.005 per trace thereafter2 | First 10k traces per month for free.
$0.005 per trace thereafter2 | Custom | +| Rate Limits | | | | +| Max ingested events / hour3 | 50,0003 / 250,000 | 500,000 | Custom | +| Total trace size storage / hour4 | 500MB3 / 2.5GB | 5GB | Custom | +| Security Controls | | | | +| Single Sign On | -- | Google
GitHub | Custom SSO | +| Deployment | Hosted in US | Hosted in US | Add-on for self-hosted
deployment in customer's VPC | +| Support | | | | +| Support Channels | Community | Email | Email
Shared Slack Channel | +| Shared Slack Channel | -- | -- | ✅ | +| Team Training | -- | -- | ✅ | +| Application Architectural Guidance | -- | -- | ✅ | +| Dedicated Customer Success Manager | -- | -- | ✅ | +| SLA | -- | -- | ✅ | +| Procurement | | | | +| Billing | Monthly, self-serve
Credit Card | Monthly, self-serve
Credit Card | Annual Invoice
ACH | +| Custom Terms and Data Privacy Agreement | -- | -- | ✅ | +| Infosec Review | -- | -- | ✅ | + +1 Seats are billed monthly on the first of the month and in the future +will be prorated if additional seats are purchased in the middle of the month. Seats +removed mid-month are not credited. +
+2 You can purchase LangSmith credits for your tracing usage. As long +as you have a valid credit card in your account, we’ll service your traces and +deduct from your credit balance. You’ll be able to set alerts and auto top-ups +on credits if you choose. +
+3 Personal accounts without a credit card on file will be rate limited +to 50,000 ingested events per hour and 500MB of storage per hour. +
+4 Trace storage includes all submitted inputs, outputs, and metadata +and is aggregated over all submission events. Depending on the design of your +application, trace data may be sent multiple times (e.g. once at the start of a +trace step and again after it is complete) + +## Questions and Answers + +### I’ve been using LangSmith since before pricing took effect for new users. When will pricing go into effect for my account? + +If you’ve been using LangSmith already, your usage will be billable starting sometime in May. At that point if you want to add seats or use more than the monthly allotment of free traces, you will need to add a credit card to LangSmith or contact sales. If you are interested in the Enterprise plan with higher rate limits and special deployment options, you can learn more or make a purchase by reaching out to [sales@langchain.dev](mailto:sales@langchain.dev). + +### Which plan is right for me? + +If you’re an individual developer, the Developer plan is a great choice for small projects. + +For teams that want to collaborate in LangSmith, check out the Plus plan. **If you are an early-stage startup building an AI application**, you may be eligible for our Startup plan with discounted prices and a generous free monthly trace allotment. Please reach out via our [Startup Contact Form](https://airtable.com/app8ZrGLtHAtFVO1o/pagfLAmdTz4ep7TGu/form) for more details. + +If you need more advanced administration, authentication and authorization, deployment options, support, or annual invoicing, the Enterprise plan is right for you. Please reach out via our [Sales Contact Form](https://www.langchain.com/contact-sales) for more details. + +### What is a seat? + +A seat is a distinct user inside your organization. We consider the total number of users (including invited users) to determine the number of seats to bill. + +### What is a trace? + +A trace is one complete invocation of your application chain or agent, evaluator run, or playground run. Here is an [example](https://smith.langchain.com/public/17c24270-9f74-47e7-b70c-d508afc448fa/r) of a single trace. + +### What is an ingested event? + +An ingested event is any distinct, trace-related data sent to LangSmith. This includes: + +- Inputs, outputs and metadata sent at the start of a run step within a trace +- Inputs, outputs and metadata sent at the end of a run step within a trace +- Feedback on run steps or traces + +### I’ve hit my rate or usage limits. What can I do? + +If you’ve consumed the monthly allotment of free traces in your account, you can add a credit card on the Developer and Plus plans to continue sending traces to LangSmith. If you’ve hit the rate limits on your tier, you can upgrade to a higher plan to get higher limits, or reach out to [support@langchain.dev](mailto:support@langchain.dev) with questions. + +### I have a developer account, can I upgrade my account to the Plus or Enterprise plan? + +Every user will have a unique personal account on the Developer plan. We cannot upgrade a Developer account to the Plus or Enterprise plans. If you’re interested in working as a team, create a separate LangSmith Organization on the Plus plan. This plan can upgraded to the Enterprise plan at a later date. + +### How will billing work? + +Seats +
+Seats are billed monthly on the first of the month in the future will be +pro-rated if additional seats are purchased in the middle of the month. Seats +removed mid-month will not be credited. +
+
+Traces +
+As long as you have a card on file in your account, we’ll service your traces and +bill you on the first of the month for traces that you submitted in the previous +month. You will be able to set usage limits if you so choose to limit the maximum +charges you could incur in any given month. + +### How can my track my usage so far this month? + +Under the Settings section for your Organization you will see subsection for Usage. There, you will able to see a graph of the daily nunber of billable LangSmith traces from the last 30, 60, or 90 days. Note that this data is delayed by 1-2 hours and so may trail your actual number of runs slightly for the current day. + +### I have a question about my bill... + +Customers on the Developer and Plus plan tiers should email [support@langchain.dev](mailto:support@langchain.dev). Customers on the Enterprise plan should contact their sales representative directly. + +Enterprise plan customers are billed annually by invoice. + +### What can I expect from Support? + +On the Developer plan, community-based support is available on [Discord](https://discord.com/invite/6adMQxSpJS). + +On the Plus plan, you will also receive preferential, email support at [support@langchain.dev](mailto:support@langchain.dev) for LangSmith-related questions only and we'll do our best to respond within the next business day. + +On the Enterprise plan, you’ll get white-glove support with a Slack channel, a dedicated customer success manager, and monthly check-ins to go over LangSmith and LangChain questions. We can help with anything from debugging, agent and RAG techniques, evaluation approaches, and cognitive architecture reviews. If you purchase the add-on to run LangSmith in your environment, we’ll also support deployments and new releases with our infra engineering team on-call. + +### Where is my data stored? + +When using LangSmith hosted at smith.langchain.com, data is stored in GCP region `us-central-1`. If you’re on the Enterprise plan, we can deliver LangSmith to run on your kubernetes cluster in AWS, GCP, or Azure so that data never leaves your environment. + +### Is LangSmith SOC 2 compliant? + +We are in the process of getting our getting our SOC 2 certification. We will update the page when we’re certified. + +### Will you train on the data that I send LangSmith? + +We will not train on your data, and you own all rights to your data. See [LangSmith Terms of Service](https://langchain.dev/terms-of-service) for more information. diff --git a/versioned_docs/version-2.0/reference.md b/versioned_docs/version-2.0/reference.md new file mode 100644 index 00000000..13d1ccb5 --- /dev/null +++ b/versioned_docs/version-2.0/reference.md @@ -0,0 +1,9 @@ +# Reference + +Technical reference that covers the LangSmith API, SDKs, data format, and architecture. + +- [ ] LangSmith Python SDK +- [ ] LangSmith TypeScript SDK +- [ ] LangSmith API Reference +- [ ] Data format +- [ ] Architecture diagram diff --git a/versioned_docs/version-2.0/release_notes.md b/versioned_docs/version-2.0/release_notes.md new file mode 100644 index 00000000..38bf0d77 --- /dev/null +++ b/versioned_docs/version-2.0/release_notes.md @@ -0,0 +1 @@ +# Release Notes diff --git a/versioned_docs/version-2.0/self_hosting/docker.mdx b/versioned_docs/version-2.0/self_hosting/docker.mdx new file mode 100644 index 00000000..cf72a085 --- /dev/null +++ b/versioned_docs/version-2.0/self_hosting/docker.mdx @@ -0,0 +1,193 @@ +--- +sidebar_label: Docker +sidebar_position: 2 +table_of_contents: true +--- + +# Self-hosting LangSmith with Docker + +:::important Enterprise License Required +Self-hosting LangSmith is an add-on to the Enterprise Plan designed for our largest, most security-conscious customers. See our [pricing page](https://www.langchain.com/pricing) for more detail, and contact us at sales@langchain.dev if you want to get a license key to trial LangSmith in your environment. +::: + +This guide provides instructions for installing and setting up your environment to run LangSmith locally using Docker. You can do this either by using the LangSmith SDK or by using Docker Compose directly. + +## Prerequisites + +1. Ensure Docker is installed and running on your system. You can verify this by running: + ```bash + docker info + ``` + If you don't see any server information in the output, make sure Docker is installed correctly and launch the Docker daemon. +2. LangSmith License Key + 1. You can get this from your Langchain representative. Contact us at sales@langchain.dev for more information. +3. LangSmith Version + 1. Our Docker Compose files are pegged to the latest self-hosted release of LangSmith. If you want to use a different version, you can specify it in the `.env` file or in the `docker-compose.yml` file. +4. OpenAI API Key(optional). + 1. Used for natural language search feature. Can specify OpenAI key in browser as well for the playground feature. +5. Oauth Configuration(optional). + 1. You can configure oauth using the `.env` file. You will need to provide a `client_id` and `client_issuer_url` for your oauth provider. + 2. Note, we do rely on the OIDC Authorization Code with PKCE flow. We currently support almost anything that is OIDC compliant however Google does not support this flow. +6. External Postgres(optional). + 1. You can configure external postgres using the `.env` file. You will need to set the `POSTGRES_DATABASE_URI` environment variable to the connection string for your postgres instance. + 2. If using a schema other than public, ensure that you do not have any other schemas with the pgcrypto extension enabled or you must include that in your search path. + 3. Note: We do only officially support Postgres versions >= 14. +7. External Redis(optional). + 1. You can configure external redis using the `.env` file. You will need to set the `REDIS_DATABASE_URI` environment variable to the connection string for your redis instance. + 2. Currently, we do not support using Redis with TLS. We will be supporting this shortly. + 3. We only official support Redis versions >= 6. + +## Running via Docker Compose + +The following explains how to run the LangSmith using Docker Compose. This is the most flexible way to run LangSmith without Kubernetes. In production, we highly recommend using Kubernetes. + +### 1. Fetch the LangSmith `docker-compose.yml` file + +You can find the `docker-compose.yml` file and related files in the LangSmith SDK repository here: [LangSmith Docker Compose File](https://github.com/langchain-ai/langsmith-sdk/blob/main/python/langsmith/cli/docker-compose.yaml) + +Copy the `docker-compose.yml` file and all files in that directory from the LangSmith SDK to your project directory. + +- Ensure that you copy the `users.xml` file as well. + +### 2. Configure environment variables + +Copy the `.env.example` file from the LangSmith SDK to your project directory and rename it to `.env`. Then, set the following environment variables in the `.env` file: + +```bash +# Don't change this file. Instead, copy it to .env and change the values there. The default values will work out of the box as long as you provide your license key. +_LANGSMITH_IMAGE_VERSION=0.2.11 # Change to your desired LangSmith version. Required. Typically you should use the latest version defined in the .env file. +LANGSMITH_LICENSE_KEY=your-license-key # Change to your Langsmith license key. Required +OPENAI_API_KEY=your-openai-api-key # Needed for Online Evals and Magic Query features +AUTH_TYPE=none # Set to "oauth" if you want to use OAuth2.0 +OAUTH_CLIENT_ID=your-client-id # Required if AUTH_TYPE=oauth +OAUTH_ISSUER_URL=https://your-issuer-url # Required if AUTH_TYPE=oauth +API_KEY_SALT=super # Change to your desired API key salt. Can be any random value. Must be set if AUTH_TYPE=oauth +POSTGRES_DATABASE_URI=postgres:postgres@langchain-db:5432/postgres # Change to your database URI if using external postgres. Otherwise, leave it as is +REDIS_DATABASE_URI=redis://langchain-redis:6379 # Change to your Redis URI if using external Redis. Otherwise, leave it as is +LOG_LEVEL=warning # Change to your desired log level +MAX_ASYNC_JOBS_PER_WORKER=10 # Change to your desired maximum async jobs per worker. We recommend 10/suggest spinning up more replicas of the queue worker if you need more throughput. +``` + +You can also set these environment variables in the `docker-compose.yml` file directly or export them in your terminal. We recommend setting them in the `.env` file. + +### 2. Start server + +Start the LangSmith application by executing the following command in your terminal: + +```bash +docker-compose up +``` + +You can also run the server in the background by running: + +```bash +docker-compose up -d +``` + +If the server starts correctly, it will open up the UI in your browser at [http://localhost](http://localhost/). + +The LangSmith UI should be visible/operational and look like this: + +![./static/langsmith_ui.png](./static/langsmith_ui.png) + +### Stopping the server + +```bash +docker-compose down +``` + +### Checking the logs + +If, at any point, you want to check if the server is running and see the logs, run + +```bash +docker-compose logs +``` + +## Running via the LangSmith SDK + +The following explains how to run the LangSmith using the LangSmith SDK. This is a convenient wrapper around the `docker-compose` command. +We recommend only using this in a local setting as it is not as flexible as using `docker-compose` directly. + +### 1. Install the LangSmith SDK + +The Python LangSmith SDKs exposes a `langsmith` command line tool. + +First, install a recent version of the `langsmith` package: + +- Python/Pip: `pip install -U langsmith` + +This will install the LangSmith Client and the bundled command line tool. + +### 2. Start server + +Start the LangSmith tracing server by executing the following command in your terminal: + +```bash +langsmith start --langsmith-license-key= --version= --openai-api-key= +``` + +If the server starts correctly, it will open up the UI in your browser at [http://localhost](http://localhost). + +The LangSmith UI should be visible/operational and look like this: + +![./static/langsmith_ui.png](./static/langsmith_ui.png) + +### Stopping the server + +To stop the server, run the following command: + +```bash +langsmith stop +``` + +### Checking the logs + +If, at any point, you want to check if the server is running and see the logs, run + +```bash +langsmith logs +``` + +## Using LangSmith + +Now that LangSmith is running, you can start using it to trace your code. You can find more information on how to use self-hosted LangSmith in the [Self-Hosted Usage Guide](/self_hosting/usage). + +### Frequently Asked Questions + +#### How can we upgrade our application? + +- We plan to release new minor versions of the LangSmith application every 6 weeks. This will include release notes and all changes should be backwards compatible. To upgrade, you will need to restart your LangSmith instance specifying the new version. + +#### How can we back up our application? + +- The docker solution uses docker volumes to store data. You can back up the data by backing up the volumes. + +#### How can we authenticate to the application? + +- Currently, our self-hosted solution supports oauth as an authn solution. +- Note, we do offer a no-auth solution but highly recommend setting up oauth before moving into production. + +#### How can I use External `Postgres` or `Redis`? + +- You can configure external postgres or redis using the external sections in the `.env` file. You will need to provide the connection url/params for the database/redis instance. Look at the `.env.example` file for more information. + +#### What networking configuration is needed for the application? + +- Our deployment only needs egress for a few things: + - Fetching images (If mirroring your images, this may not be needed) + - Talking to any LLMs +- Your VPC can set up rules to limit any other access. +- Note: We require the X-Tenant-Id to be allowed to be passed through to the backend service. This is used to determine which tenant the request is for. + +#### What resources should we allocate to the application? + +- We recommend at least 4 vCPUs and 16GB of memory for our application. This is a rough estimate and can vary based on the number of users and the size of the data you are working with. + +#### Increasing the number of replicas + +- If you need more throughput, you can increase the number of replicas of the queue worker by running the following command: + ```bash + docker-compose up --scale langchain-queue=5 + ``` + This will start 5 replicas of the queue worker. You can change the number to whatever you need. Note that these containers are fairly CPU intensive, and you should ensure you have enough resources to support the number of replicas you are starting. diff --git a/versioned_docs/version-2.0/self_hosting/kubernetes.mdx b/versioned_docs/version-2.0/self_hosting/kubernetes.mdx new file mode 100644 index 00000000..d4e84a2f --- /dev/null +++ b/versioned_docs/version-2.0/self_hosting/kubernetes.mdx @@ -0,0 +1,197 @@ +--- +sidebar_label: Kubernetes +sidebar_position: 1 +table_of_contents: true +--- + +# Self-hosting LangSmith on Kubernetes + +:::important Enterprise License Required +Self-hosting LangSmith is an add-on to the Enterprise Plan designed for our largest, most security-conscious customers. See our [pricing page](https://www.langchain.com/pricing) for more detail, and contact us at sales@langchain.dev if you want to get a license key to trial LangSmith in your environment. +::: + +This guide will walk you through the process of deploying LangSmith to a Kubernetes cluster. We will use Helm to install LangSmith and its dependencies. + +We've successfully tested LangSmith on the following Kubernetes distributions: + +- Google Kubernetes Engine (GKE) +- Amazon Elastic Kubernetes Service (EKS) +- Azure Kubernetes Service (AKS) +- OpenShift +- Minikube and Kind (for development purposes) + +## Prerequisites + +Ensure you have the following tools/items ready. Some items are marked optional but : + +1. A working Kubernetes cluster that you can access via `kubectl`. Your cluster should have the following minimum requirements: + + 1. Recommended: At least 4 vCPUs, 16GB Memory available + - You may need to tune resource requests/limits for all of our different services based off of organization size/usage + 2. Valid Dynamic PV provisioner or PVs available on your cluster. You can verify this by running: + ```bash + kubectl get storageclass + ``` + + The output should show at least one storage class with a provisioner that supports dynamic provisioning. For example: + + ```bash + NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE + gp2 (default) kubernetes.io/aws-ebs Delete WaitForFirstConsumer false 161d + ``` + +2. Helm + 1. `brew install helm` +3. LangSmith License Key + 1. You can get this from your Langchain representative. Contact us at sales@langchain.dev for more information. +4. SSL(optional) + 1. This should be attachable to a load balancer that will be provisioned by your cloud provider. This will be used for the frontend service. +5. OpenAI API Key(optional). + 1. Used for natural language search feature(beta). Can specify OpenAI key in browser as well for the playground feature. +6. OAuth Configuration(optional). + 1. You can configure oauth using the `values.yaml` file. You will need to provide a `client_id` and `client_issuer_url` for your OAuth provider. + 2. Note, we do rely on the OIDC Authorization Code with PKCE flow. We currently support almost anything that is OIDC compliant however Google does not support this flow. + 3. Without OAuth, you will not be able to create users or organizations. +7. External Postgres(optional). + 1. You can configure external postgres using the `values.yaml` file. You will need to provide connection parameters for your postgres instance. + 2. If using a schema other than public, ensure that you do not have any other schemas with the pgcrypto extension enabled, or you must include that in your search path. + 3. Note: We do only officially support Postgres versions >= 14. +8. External Redis(optional). + 1. You can configure external redis using the `values.yaml` file. You will need to provide a connection url for your redis instance. + 2. Currently, we do not support using Redis with TLS. We will be supporting this shortly. + 3. We only official support Redis versions >= 6. + +## Configure your Helm Charts: + +1. Create a new file called `langsmith_config.yaml`. This should have a similar structure to the `values.yaml` file in the LangSmith Helm Chart repository. Only include the values you want to override to avoid having to update the file every time the chart is updated. +2. Override any values in the file. Refer to the documentation for the [LangSmith Helm Chart](https://github.com/langchain-ai/helm/tree/main/charts/langsmith) to see all configurable values. Some values we recommend setting: + 1. Resources + 2. SSL(If on EKS or some other cloud provider) + 1. Add an annotation to the `frontend.service` object to tell your cloud provider to provision a load balancer with said certificate attached + 3. OpenAI Api Key + 4. Images + 5. Oauth + +An example bare minimum config file `langsmith_config.yaml`: + +```yaml +config: + langsmithLicenseKey: "" +``` + +You can also see some example configurations in the examples directory. + +## Deploying to Kubernetes: + +1. Verify that you can connect to your Kubernetes cluster(note: We highly suggest installing into an empty namespace) + + 1. Run `kubectl get pods` + + Output should look something like: + + ```bash + kubectl get pods ⎈ langsmith-eks-2vauP7wf 21:07:46 + No resources found in default namespace. + ``` + +2. Ensure you have the Langchain Helm repo added. (skip this step if you are using local charts) + + helm repo add langchain https://langchain-ai.github.io/helm/ + "langchain" has been added to your repositories + +3. Run `helm install langsmith langchain/langsmith --values langsmith_config.yaml --namespace --version ` + + Output should look something like: + + ```bash + NAME: langsmith + LAST DEPLOYED: Fri Sep 17 21:08:47 2021 + NAMESPACE: langsmith + STATUS: deployed + REVISION: 1 + TEST SUITE: None + ``` + +4. Run `kubectl get pods` + Output should now look something like: + + ```bash + langsmith-backend-6ff46c99c4-wz22d 1/1 Running 0 3h2m + langsmith-frontend-6bbb94c5df-8xrlr 1/1 Running 0 3h2m + langsmith-hub-backend-5cc68c888c-vppjj 1/1 Running 0 3h2m + langsmith-playground-6d95fd8dc6-x2d9b 1/1 Running 0 3h2m + langsmith-postgres-0 1/1 Running 0 9h + langsmith-queue-5898b9d566-tv6q8 1/1 Running 0 3h2m + langsmith-redis-0 1/1 Running 0 9h + ``` + +## Validate your deployment: + +1. Run `kubectl get services` + + Output should look something like: + + ```bash + NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE + langsmith-backend ClusterIP 172.20.140.77 1984/TCP 35h + langsmith-frontend LoadBalancer 172.20.253.251 80:31591/TCP 35h + langsmith-hub-backend ClusterIP 172.20.112.234 1985/TCP 35h + langsmith-playground ClusterIP 172.20.153.194 3001/TCP 9h + langsmith-postgres ClusterIP 172.20.244.82 5432/TCP 35h + langsmith-redis ClusterIP 172.20.81.217 6379/TCP 35h + ``` + +2. Curl the external ip of the `langsmith-frontend` service: + + ```bash + curl /api/tenants + [{"id":"00000000-0000-0000-0000-000000000000","has_waitlist_access":true,"created_at":"2023-09-13T18:25:10.488407","display_name":"Personal","config":{"is_personal":true,"max_identities":1},"tenant_handle":"default"}]% + ``` + +3. Visit the external ip for the `langsmith-frontend` service on your browser + + The LangSmith UI should be visible/operational + + ![./static/langsmith_ui.png](./static/langsmith_ui.png) + +## Using LangSmith + +Now that LangSmith is running, you can start using it to trace your code. You can find more information on how to use self-hosted LangSmith in the [Self-Hosted Usage Guide](/self_hosting/usage). + +### Frequently Asked Questions: + +#### How can we upgrade our application? + +- We plan to release new minor versions of the LangSmith application every 6 weeks. This will include release notes and all changes should be backwards compatible. To upgrade, you will need to follow the upgrade instructions in the Helm README and run a `helm upgrade langsmith --values ` + +#### How can we back up our application? + +- Currently, we rely on PVCs/PV to power storage for our application. We strongly encourage setting up `Persistent Volume` backups or moving to a managed service for `Postgres` to support disaster recovery + +#### How does load balancing work/ingress work? + +- Currently, our application spins up one load balancer using a k8s service of type `LoadBalancer` for our frontend. If you do not want to set up a load balancer you can simply port-forward the frontend and use that as your external ip for the application. We also have an option for the chart to provision an ingress resource for the application. + +#### How can we authenticate to the application? + +- Currently, our self-hosted solution supports oauth as an authn solution. Note, we do offer a no-auth solution but highly recommend setting up oauth before moving into production. + +#### How can I use External `Postgres` or `Redis`? + +- You can configure external postgres or redis using the external sections in the `values.yaml` file. You will need to provide the connection url/params for the database/redis instance. Look at the configuration above example for more information. + +#### What networking configuration is needed for the application? + +Our deployment only needs egress for a few things: + +- Fetching images (If mirroring your images, this may not be needed) +- Talking to any LLMs +- Talking to any external services you may have configured +- Fetching OAuth information + Your VPC can set up rules to limit any other access. Note: We require the X-Tenant-Id to be allowed to be passed through to the backend service. This is used to determine which tenant the request is for. + +#### What resources should we allocate to the application? + +- We recommend at least 4 vCPUs and 16GB of memory for our application. +- We have some default resources set in our `values.yaml` file. You can override these values to tune resource usage for your organization. +- If the metrics server is enabled in your cluster, we also recommend enabling autoscaling on all deployments. diff --git a/versioned_docs/version-2.0/self_hosting/release_notes.mdx b/versioned_docs/version-2.0/self_hosting/release_notes.mdx new file mode 100644 index 00000000..815bdcb9 --- /dev/null +++ b/versioned_docs/version-2.0/self_hosting/release_notes.mdx @@ -0,0 +1,99 @@ +--- +sidebar_label: Release Notes (Self-Hosted) +sidebar_position: 4 +--- + +# LangSmith Release Notes + +:::note +If you are updating directly from LangSmith v0.1.x and you wish to retain access to run data in the Langsmith UI after updating, you must first update to v0.2.x and perform a data migration. **Updating directly from v0.1.x to v0.3.x or later will result in data loss as the `runs` table in postgres will be dropped when deploying LangSmith v0.3.x or higher.** Details are available at https://github.com/langchain-ai/helm/blob/main/charts/langsmith/docs/UPGRADE-0.2.x.md +::: + +## Week of March 25, 2024 - LangSmith v0.4 + +LangSmith 0.4 improves performance and reliability, implements a new asynchronous queue worker to optimize run ingests, and an API key salt parameter. + +### Breaking changes + +- This release adds an API key salt parameter. This previously defaulted to your LangSmith License Key. **For updates from earlier versions you should set this parameter to your license key to ensure backwards compatibility.** Using a new api key salt will invalidate all existing api keys. +- This release makes Clickhouse persistence use 50Gi of storage by default. You can adjust this by changing the `clickhouse.statefulSet.persistence.size` value in your `values.yaml` file. + - If your existing configuration does not configure persistence already, you will need to resize your existing pvc or set `clickhouse.statefulSet.persistence.size` to the previous default value of `8Gi`. + +### Performance and Reliability Changes + +- Implemented a new asynchronous queue worker and cached token encodings to improve performance when ingesting traces, reducing the delay between ingest and display in the LangSmith UI. + +### Infrastructure changes + +- Some our image repositories have been updated. You can see the root repositories in our `values.yaml` file and may need to update mirrors to pick up the new images. +- Clickhouse persistence now uses 50Gi of storage by default. You can adjust this by changing the `clickhouse.statefulSet.persistence.size` value in your `values.yaml` file. + - If your existing configuration cannot support 50Gi, you may need to resize your existing storage class or set `clickhouse.statefulSet.persistence.size` to the previous default value of `8Gi`. +- Consolidation of hubBackend and backend services. We now use one service to serve both of these endpoints. This should not impact your application. + +### Admin changes + +- Added an API key salt parameter in `values.yml`. This can be set to a custom value and changing it will invalidate all existing api keys. +- Changed the OAuth flow to leverage Access Tokens instead of OIDC ID tokens. This change should not impact the end user experience. +- Added scripts to enable feature flags in self-hosted environments for use in previewing pre-release features. Details are available at https://github.com/langchain-ai/helm/blob/main/charts/langsmith/docs/ADD-FEATURE-FLAG.md + +### Deprecation notices + +With the release of 0.4: + +- LangSmith 0.3.x and earlier are now in maintenance mode and may only receive critical security fixes. + +## Week of Februrary 21, 2024 - LangSmith v0.3 + +LangSmith 0.3 improves performance and reliability, adds improved monitoring charts group by metadata and tag, and adds cost tracking. + +### Breaking changes + +- This release will drop the postgres run tables - if you are making a migration from LangSmith v0.1 and wish to retain run data, you must first update to v0.2 and perform a data migration. See https://github.com/langchain-ai/helm/blob/main/charts/langsmith/docs/UPGRADE-0.2.x.md for additional details + +### Performance and Reliability Changes + +- Continued performance when ingesting traces, reducing the delay between ingest and display in the LangSmith UI. + +### Admin changes + +- None + +### Deprecation notices + +With the release of 0.3: + +- LangSmith 0.2.x and earlier are now in maintenance mode and may only receive critical security fixes. + +## Week of January 29, 2024 - LangSmith v0.2 + +LangSmith 0.2 improves performance and reliability, adds a updated interface for reviewing trace data, and adds support for batch processing of traces. + +### Requirements + +- This release requires `langsmith-sdk` version ≥ `0.0.71` (Python) and ≥ `0.0.56` (JS/TS) to support changes in pagination of API results. Older versions will only return the first 100 results when querying an endpoint. + +### Breaking changes + +- The search syntax for metadata in runs has changed and limits support for nested JSON to a single level. If you are supplying custom metadata in traces, you should flatten your metadata structure in order to allow it to be searchable, (e.g. `{"user_id": ..., "user_name":...,}`) and then search using `has(metadata, '{"user_name": ...}')` + +### Performance and Reliability Changes + +- Improved performance when ingesting traces, reducing the delay between ingest and display in the LangSmith UI. +- Improved performance for updates and deletes on annotation labels. +- Added pagination of API responses. +- Fixed an issue impacting natural language searches. + +### Infrastructure Changes + +- Added the `clickhouse` database service. Run results will now be stored in ClickHouse instead of Postgres to improve performance and scalability and reduce delays in the time it takes for runs to appear in LangSmith. + - Note that if you wish to retain access to run data in the Langsmith UI after updating, a data migration will need to be performed. Details are available at https://github.com/langchain-ai/helm/blob/main/charts/langsmith/docs/UPGRADE-0.2.x.md + +### Admin changes + +- Increased the maximum number of users per organization from 5 to 100 for new organizations. + +### Deprecation notices + +With the release of 0.2: + +- LangSmith 0.1.x is now in maintenance mode and may only receive critical security fixes. diff --git a/versioned_docs/version-2.0/self_hosting/static/langsmith_ui.png b/versioned_docs/version-2.0/self_hosting/static/langsmith_ui.png new file mode 100644 index 00000000..2a229dbc Binary files /dev/null and b/versioned_docs/version-2.0/self_hosting/static/langsmith_ui.png differ diff --git a/versioned_docs/version-2.0/self_hosting/usage.mdx b/versioned_docs/version-2.0/self_hosting/usage.mdx new file mode 100644 index 00000000..4263a7a5 --- /dev/null +++ b/versioned_docs/version-2.0/self_hosting/usage.mdx @@ -0,0 +1,47 @@ +--- +sidebar_label: Usage +sidebar_position: 3 +table_of_contents: true +--- + +# How to use your self-hosted instance of LangSmith + +This guide will walk you through the process of using your self-hosted instance of LangSmith. + +:::important Self-Hosted LangSmith Instance Required +This guide assumes you have already deployed a self-hosted LangSmith instance. If you have not, please refer to the [kubernetes deployment guide](/self_hosting/kubernetes) or the [docker deployment guide](/self_hosting/docker). +::: + +### Using your deployment: + +1. Once you have deployed your instance, you can access the LangSmith UI at `http://`. +2. The backend API will be available at `http:///api` and the hub API will be available at `http:///api-hub`. + +To use the LangSmith API, you will need to set the following environment variables in your application: + +```bash +LANGCHAIN_ENDPOINT=http:///api +LANGCHAIN_HUB_API_URL=http:///api-hub +LANGCHAIN_API_KEY=foo # Set to a legitimate API key if using OAuth +``` + +You can also configure these variables directly in the LangSmith SDK client: + +```python +import langsmith + +langsmith_client = langsmith.Client( + api_key='', + api_url='http:///api', +) + +import langchainhub + +langchainhub.Client( + api_key='', + api_url='http:///api-hub' +) +``` + +After setting the above, you should be able to run your code and see the results in your self-hosted instance. +We recommend running through the [quickstart guide](https://docs.smith.langchain.com/#quick-start) to get a feel for how to use LangSmith. diff --git a/versioned_docs/version-2.0/static/LangSmith_Diagram-GA-final.png b/versioned_docs/version-2.0/static/LangSmith_Diagram-GA-final.png new file mode 100644 index 00000000..de867c8a Binary files /dev/null and b/versioned_docs/version-2.0/static/LangSmith_Diagram-GA-final.png differ diff --git a/versioned_docs/version-2.0/static/rendered_trace.png b/versioned_docs/version-2.0/static/rendered_trace.png new file mode 100644 index 00000000..b98c552e Binary files /dev/null and b/versioned_docs/version-2.0/static/rendered_trace.png differ diff --git a/versioned_docs/version-2.0/tutorials.md b/versioned_docs/version-2.0/tutorials.md new file mode 100644 index 00000000..90f62b4a --- /dev/null +++ b/versioned_docs/version-2.0/tutorials.md @@ -0,0 +1,14 @@ +# Tutorials + +New to LangSmith? This is the place to start. Here, you'll find a hands-on introduction to key LangSmith workflows. + +- Build up datasets from production traces +- Iterate on prompts +- Create a data flywheel to optimize your LLM application +- Debug your LLM application +- Utilize LLM-as-a-judge evaluators +- Evaluate RAG applications +- Prompt iteration as a non-developer +- Reduce noise during evaluation +- Evaluate agents +- Back testing diff --git a/versioned_sidebars/version-2.0-sidebars.json b/versioned_sidebars/version-2.0-sidebars.json new file mode 100644 index 00000000..85e13c5a --- /dev/null +++ b/versioned_sidebars/version-2.0-sidebars.json @@ -0,0 +1,27 @@ +{ + "sidebar": [ + "index", + "tutorials", + "how_to_guides", + "concepts", + "reference", + "pricing", + { + "type": "category", + "label": "Self-Hosting", + "collapsed": true, + "collapsible": true, + "items": [ + { + "type": "autogenerated", + "dirName": "self_hosting" + } + ], + "link": { + "type": "generated-index", + "description": "Self-hosting LangSmith requires an enterprise license. Check out the guides below for more information." + } + }, + "release_notes" + ] +} diff --git a/versions.json b/versions.json new file mode 100644 index 00000000..59f17b2b --- /dev/null +++ b/versions.json @@ -0,0 +1,3 @@ +[ + "2.0" +]