Merge remote-tracking branch 'upstream/main' into cache

docs/_scripts/generate_api_reference_links.py

@@ -6,6 +6,9 @@
 from typing import List, Literal, Optional
 from typing_extensions import TypedDict
 
+
+from functools import lru_cache
+
 import nbformat
 from nbconvert.preprocessors import Preprocessor
 
@@ -47,6 +50,8 @@
     (["langgraph.graph"], "langgraph.constants", "END", "constants"),
     (["langgraph.constants"], "langgraph.types", "Send", "types"),
     (["langgraph.constants"], "langgraph.types", "Interrupt", "types"),
+    (["langgraph.constants"], "langgraph.types", "interrupt", "types"),
+    (["langgraph.constants"], "langgraph.types", "Command", "types"),
     ([], "langgraph.types", "RetryPolicy", "types"),
     ([], "langgraph.checkpoint.base", "Checkpoint", "checkpoints"),
     ([], "langgraph.checkpoint.base", "CheckpointMetadata", "checkpoints"),
@@ -83,8 +88,11 @@ def _make_regular_expression(pkg_prefix: str) -> re.Pattern:
 _IMPORT_LANGGRAPH_RE = _make_regular_expression("langgraph")
 
 
-def _get_full_module_name(module_path, class_name) -> Optional[str]:
-    """Get full module name using inspect"""
+
+
+@lru_cache(maxsize=10_000)
+def _get_full_module_name(module_path: str, class_name: str) -> Optional[str]:
+    """Get full module name using inspect, with LRU cache to memoize results."""
     try:
         module = importlib.import_module(module_path)
         class_ = getattr(module, class_name)
@@ -95,13 +103,12 @@ def _get_full_module_name(module_path, class_name) -> Optional[str]:
             return module_path
         return module.__name__
     except AttributeError as e:
-        logger.warning(f"Could not find module for {class_name}, {e}")
+        logger.warning(f"API Reference: Could not find module for {class_name}, {e}")
         return None
     except ImportError as e:
-        logger.warning(f"Failed to load for class {class_name}, {e}")
+        logger.warning(f"API Reference: Failed to load for class {class_name}, {e}")
         return None
 
-
 def _get_doc_title(data: str, file_name: str) -> str:
     try:
         return re.findall(r"^#\s*(.*)", data, re.MULTILINE)[0]
@@ -115,10 +122,10 @@ def _get_doc_title(data: str, file_name: str) -> str:
 
 
 class ImportInformation(TypedDict):
-    imported: str  # imported class name
-    source: str  # module path
-    docs: str  # URL to the documentation
-    title: str  # Title of the document
+    imported: str  # The name of the class that was imported.
+    source: str  # The full module path from which the class was imported.
+    docs: str  # The URL pointing to the class's documentation.
+    title: str  # The title of the document where the import is used.
 
 
 def _get_imports(
@@ -211,36 +218,73 @@ def _get_imports(
     return imports
 
 
-class ImportPreprocessor(Preprocessor):
-    """A preprocessor to replace imports in each Python code cell with links to their
-    documentation and append the import info in a comment."""
+def get_imports(code: str, doc_title: str) -> List[ImportInformation]:
+    """Retrieve all import references from the given code for specified ecosystems.
 
-    def preprocess(self, nb, resources):
-        self.all_imports = []
-        file_name = os.path.basename(resources.get("metadata", {}).get("name", ""))
-        _DOC_TITLE = _get_doc_title(nb.cells[0].source, file_name)
+    Args:
+        code: The source code from which to extract import references.
+        doc_title: The documentation title associated with the code.
 
-        cells = []
-        for cell in nb.cells:
-            if cell.cell_type == "code":
-                cells.append(cell)
-                imports = _get_imports(
-                    cell.source, _DOC_TITLE, "langchain"
-                ) + _get_imports(cell.source, _DOC_TITLE, "langgraph")
-                if not imports:
-                    continue
+    Returns:
+        A list of import information for each import found.
+    """
+    ecosystems = ["langchain", "langgraph"]
+    all_imports = []
+    for package_ecosystem in ecosystems:
+        all_imports.extend(_get_imports(code, doc_title, package_ecosystem))
+    return all_imports
 
-                cells.append(
-                    nbformat.v4.new_markdown_cell(
-                        source=f"""
-<div>
-<b>API Reference:</b>
-{' | '.join(f'<a href="{imp["docs"]}">{imp["imported"]}</a>' for imp in imports)}
-</div>
-                    """
-                    )
-                )
-            else:
-                cells.append(cell)
-        nb.cells = cells
-        return nb, resources
+
+def update_markdown_with_imports(markdown: str) -> str:
+    """Update markdown to include API reference links for imports in Python code blocks.
+
+    This function scans the markdown content for Python code blocks, extracts any imports, and appends links to their API documentation.
+
+    Args:
+        markdown: The markdown content to process.
+
+    Returns:
+        Updated markdown with API reference links appended to Python code blocks.
+
+    Example:
+        Given a markdown with a Python code block:
+
+        ```python
+        from langchain.nlp import TextGenerator
+        ```
+        This function will append an API reference link to the `TextGenerator` class from the `langchain.nlp` module if it's recognized.
+    """
+    code_block_pattern = re.compile(
+        r'(?P<indent>[ \t]*)```(?P<language>python|py)\n(?P<code>.*?)\n(?P=indent)```', re.DOTALL
+    )
+
+    def replace_code_block(match: re.Match) -> str:
+        """Replace the matched code block with additional API reference links if imports are found.
+
+        Args:
+            match (re.Match): The regex match object containing the code block.
+
+        Returns:
+            str: The modified code block with API reference links appended if applicable.
+        """
+        indent = match.group('indent')
+        code_block = match.group('code')
+        language = match.group('language')  # Preserve the language from the regex match
+        # Retrieve import information from the code block
+        imports = get_imports(code_block, "__unused__")
+
+        original_code_block = match.group(0)
+        # If no imports are found, return the original code block
+        if not imports:
+            return original_code_block
+
+        # Generate API reference links for each import
+        api_links = ' | '.join(
+            f'<a href="{imp["docs"]}">{imp["imported"]}</a>' for imp in imports
+        )
+        # Return the code block with appended API reference links
+        return f'{original_code_block}\n\n{indent}API Reference: {api_links}'
+
+    # Apply the replace_code_block function to all matches in the markdown
+    updated_markdown = code_block_pattern.sub(replace_code_block, markdown)
+    return updated_markdown

docs/_scripts/notebook_convert.py

@@ -6,8 +6,6 @@
 from nbconvert.exporters import MarkdownExporter
 from nbconvert.preprocessors import Preprocessor
 
-from generate_api_reference_links import ImportPreprocessor
-
 
 class EscapePreprocessor(Preprocessor):
     def preprocess_cell(self, cell, resources, cell_index):
@@ -107,7 +105,6 @@ def preprocess_cell(self, cell, resources, cell_index):
     preprocessors=[
         EscapePreprocessor,
         ExtractAttachmentsPreprocessor,
-        ImportPreprocessor,
     ],
     template_name="mdoutput",
     extra_template_basedirs=[

docs/_scripts/notebook_hooks.py

@@ -1,10 +1,13 @@
 import logging
 import os
+import re
 from typing import Any, Dict
 
-from mkdocs.structure.pages import Page
 from mkdocs.structure.files import Files, File
+from mkdocs.structure.pages import Page
+
 from notebook_convert import convert_notebook
+from generate_api_reference_links import update_markdown_with_imports
 
 logger = logging.getLogger(__name__)
 logging.basicConfig()
@@ -35,12 +38,83 @@ def on_files(files: Files, **kwargs: Dict[str, Any]):
     return new_files
 
 
+def _highlight_code_blocks(markdown: str) -> str:
+    """Find code blocks with highlight comments and add hl_lines attribute.
+
+    Args:
+        markdown: The markdown content to process.
+
+    Returns:
+        updated Markdown code with code blocks containing highlight comments
+        updated to use the hl_lines attribute.
+    """
+    # Pattern to find code blocks with highlight comments and without
+    # existing hl_lines for Python and JavaScript
+    # Pattern to find code blocks with highlight comments, handling optional indentation
+    code_block_pattern = re.compile(
+        r"(?P<indent>[ \t]*)```(?P<language>py|python|js|javascript)(?!\s+hl_lines=)\n"
+        r"(?P<code>((?:.*\n)*?))"  # Capture the code inside the block using named group
+        r"(?P=indent)```"  # Match closing backticks with the same indentation
+    )
+
+    def replace_highlight_comments(match: re.Match) -> str:
+        indent = match.group("indent")
+        language = match.group("language")
+        code_block = match.group("code")
+        lines = code_block.split("\n")
+        highlighted_lines = []
+
+        # Skip initial empty lines
+        while lines and not lines[0].strip():
+            lines.pop(0)
+
+        lines_to_keep = []
+
+        comment_syntax = (
+            "# highlight-next-line"
+            if language in ["py", "python"]
+            else "// highlight-next-line"
+        )
+
+        for line in lines:
+            if comment_syntax in line:
+                count = len(lines_to_keep) + 1
+                highlighted_lines.append(str(count))
+            else:
+                lines_to_keep.append(line)
+
+        # Reconstruct the new code block
+        new_code_block = "\n".join(lines_to_keep)
+
+        if highlighted_lines:
+            return (
+                f'{indent}```{language} hl_lines="{" ".join(highlighted_lines)}"\n'
+                # The indent and terminating \n is already included in the code block
+                f'{new_code_block}'
+                f'{indent}```'
+            )
+        else:
+            return (
+                f"{indent}```{language}\n"
+                # The indent and terminating \n is already included in the code block
+                f"{new_code_block}"
+                f"{indent}```"
+            )
+
+    # Replace all code blocks in the markdown
+    markdown = code_block_pattern.sub(replace_highlight_comments, markdown)
+    return markdown
+
+
 def on_page_markdown(markdown: str, page: Page, **kwargs: Dict[str, Any]):
     if DISABLED:
         return markdown
     if page.file.src_path.endswith(".ipynb"):
         logger.info("Processing Jupyter notebook: %s", page.file.src_path)
-        body = convert_notebook(page.file.abs_src_path)
-        return body
+        markdown = convert_notebook(page.file.abs_src_path)
 
+    # Append API reference links to code blocks
+    markdown = update_markdown_with_imports(markdown)
+    # Apply highlight comments to code blocks
+    markdown = _highlight_code_blocks(markdown)
     return markdown

docs/docs/cloud/deployment/cloud.md

@@ -5,7 +5,7 @@ LangGraph Cloud is available within <a href="https://www.langchain.com/langsmith
 ## Prerequisites
 
 1. LangGraph Cloud applications are deployed from GitHub repositories. Configure and upload a LangGraph Cloud application to a GitHub repository in order to deploy it to LangGraph Cloud.
-1. [Verify that the LangGraph API runs locally](test_locally.md). If the API does not build and run successfully (i.e. `langgraph up`), deploying to LangGraph Cloud will fail as well.
+1. [Verify that the LangGraph API runs locally](test_locally.md). If the API does not run successfully (i.e. `langgraph dev`), deploying to LangGraph Cloud will fail as well.
 
 ## Create New Deployment
 

docs/docs/cloud/deployment/test_locally.md

@@ -6,17 +6,11 @@ Testing locally ensures that there are no errors or conflicts with Python depend
 
 ## Setup
 
-Install the proper packages:
+Install the LangGraph CLI package:
 
-
-=== "pip" 
-    ```bash
-    pip install -U langgraph-cli
-    ```
-=== "Homebrew (macOS only)"
-    ```bash
-    brew install langgraph-cli
-    ```
+```bash
+pip install -U "langgraph-cli[inmem]"
+```
 
 Ensure you have an API key, which you can create from the [LangSmith UI](https://smith.langchain.com) (Settings > API Keys). This is required to authenticate that you have LangGraph Cloud access. After you have saved the key to a safe place, place the following line in your `.env` file:
 
@@ -29,16 +23,26 @@ LANGSMITH_API_KEY = *********
 Once you have installed the CLI, you can run the following command to start the API server for local testing:
 
 ```shell
-langgraph up
+langgraph dev
 ```
 
 This will start up the LangGraph API server locally. If this runs successfully, you should see something like:
 
-```shell
-Ready!
-- API: http://localhost:8123
-2024-06-26 19:20:41,056:INFO:uvicorn.access 127.0.0.1:44138 - "GET /ok HTTP/1.1" 200
-```
+>    Ready!
+> 
+>    - API: [http://localhost:2024](http://localhost:2024/)
+>     
+>    - Docs: http://localhost:2024/docs
+>     
+>    - LangGraph Studio Web UI: https://smith.langchain.com/studio/?baseUrl=http://127.0.0.1:2024
+
+!!! note "In-Memory Mode"
+
+    The `langgraph dev` command starts LangGraph Server in an in-memory mode. This mode is suitable for development and testing purposes. For production use, you should deploy LangGraph Server with access to a persistent storage backend.
+
+    If you want to test your application with a persistent storage backend, you can use the `langgraph up` command instead of `langgraph dev`. You will
+    need to have `docker` installed on your machine to use this command.
+
 
 ### Interact with the server
 
@@ -53,7 +57,7 @@ You can either initialize by passing authentication or by setting an environment
     ```python
     from langgraph_sdk import get_client
 
-    # only pass the url argument to get_client() if you changed the default port when calling langgraph up
+    # only pass the url argument to get_client() if you changed the default port when calling langgraph dev
     client = get_client(url=<DEPLOYMENT_URL>,api_key=<LANGSMITH_API_KEY>)
     # Using the graph deployed with the name "agent"
     assistant_id = "agent"
@@ -65,7 +69,7 @@ You can either initialize by passing authentication or by setting an environment
     ```js
     import { Client } from "@langchain/langgraph-sdk";
 
-    // only set the apiUrl if you changed the default port when calling langgraph up
+    // only set the apiUrl if you changed the default port when calling langgraph dev
     const client = new Client({ apiUrl: <DEPLOYMENT_URL>, apiKey: <LANGSMITH_API_KEY> });
     // Using the graph deployed with the name "agent"
     const assistantId = "agent";
@@ -91,7 +95,7 @@ If you have a `LANGSMITH_API_KEY` set in your environment, you do not need to ex
     ```python
     from langgraph_sdk import get_client
 
-    # only pass the url argument to get_client() if you changed the default port when calling langgraph up
+    # only pass the url argument to get_client() if you changed the default port when calling langgraph dev
     client = get_client()
     # Using the graph deployed with the name "agent"
     assistant_id = "agent"
@@ -103,7 +107,7 @@ If you have a `LANGSMITH_API_KEY` set in your environment, you do not need to ex
     ```js
     import { Client } from "@langchain/langgraph-sdk";
 
-    // only set the apiUrl if you changed the default port when calling langgraph up
+    // only set the apiUrl if you changed the default port when calling langgraph dev
     const client = new Client();
     // Using the graph deployed with the name "agent"
     const assistantId = "agent";