Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs: lint #2862

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,13 @@ clean-docs:

## Run format against the project documentation.
format-docs:
poetry run ruff format docs/docs
poetry run ruff check --fix docs/docs
poetry run ruff format docs/
poetry run ruff check --fix docs/

# Check the docs for linting violations
lint-docs:
poetry run ruff format --check docs/docs
poetry run ruff check docs/docs
poetry run ruff format --check docs
poetry run ruff check docs/

codespell:
./docs/codespell_notebooks.sh .
Expand Down
2 changes: 1 addition & 1 deletion docs/_scripts/download_tiktoken.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

# This will trigger the download and caching of the necessary files
for encoding in ("gpt2", "gpt-3.5"):
tiktoken.encoding_for_model(encoding)
tiktoken.encoding_for_model(encoding)
15 changes: 8 additions & 7 deletions docs/_scripts/generate_api_reference_links.py
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,8 @@ def update_markdown_with_imports(markdown: str) -> str:
This function will append an API reference link to the `TextGenerator` class from the `langchain.nlp` module if it's recognized.
"""
code_block_pattern = re.compile(
r'(?P<indent>[ \t]*)```(?P<language>python|py)\n(?P<code>.*?)\n(?P=indent)```', re.DOTALL
r"(?P<indent>[ \t]*)```(?P<language>python|py)\n(?P<code>.*?)\n(?P=indent)```",
re.DOTALL,
)

def replace_code_block(match: re.Match) -> str:
Expand All @@ -262,9 +263,9 @@ def replace_code_block(match: re.Match) -> str:
Returns:
str: The modified code block with API reference links appended if applicable.
"""
indent = match.group('indent')
code_block = match.group('code')
language = match.group('language') # Preserve the language from the regex match
indent = match.group("indent")
code_block = match.group("code")
language = match.group("language") # Preserve the language from the regex match
# Retrieve import information from the code block
imports = get_imports(code_block, "__unused__")

Expand All @@ -274,12 +275,12 @@ def replace_code_block(match: re.Match) -> str:
return original_code_block

# Generate API reference links for each import
api_links = ' | '.join(
api_links = " | ".join(
f'<a href="{imp["docs"]}">{imp["imported"]}</a>' for imp in imports
)
# Return the code block with appended API reference links
return f'{original_code_block}\n\n{indent}API Reference: {api_links}'
return f"{original_code_block}\n\n{indent}API Reference: {api_links}"

# Apply the replace_code_block function to all matches in the markdown
updated_markdown = code_block_pattern.sub(replace_code_block, markdown)
return updated_markdown
return updated_markdown
23 changes: 13 additions & 10 deletions docs/_scripts/prepare_notebooks_for_ci.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import nbformat

logger = logging.getLogger(__name__)
NOTEBOOK_DIRS = ("docs/docs/how-tos","docs/docs/tutorials")
NOTEBOOK_DIRS = ("docs/docs/how-tos", "docs/docs/tutorials")
DOCS_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CASSETTES_PATH = os.path.join(DOCS_PATH, "cassettes")

Expand All @@ -20,7 +20,7 @@

NOTEBOOKS_NO_CASSETTES = (
"docs/docs/how-tos/visualization.ipynb",
"docs/docs/how-tos/many-tools.ipynb"
"docs/docs/how-tos/many-tools.ipynb",
)

NOTEBOOKS_NO_EXECUTION = [
Expand All @@ -43,7 +43,7 @@
"docs/docs/tutorials/lats/lats.ipynb", # issues only when running with VCR
"docs/docs/tutorials/rag/langgraph_crag.ipynb", # flakiness from tavily
"docs/docs/tutorials/rag/langgraph_adaptive_rag.ipynb", # Cannot create a consistent method resolution error from VCR
"docs/docs/how-tos/map-reduce.ipynb" # flakiness from structured output, only when running with VCR
"docs/docs/how-tos/map-reduce.ipynb", # flakiness from structured output, only when running with VCR
]


Expand Down Expand Up @@ -71,9 +71,9 @@ def is_comment(code: str) -> bool:


def has_blocklisted_command(code: str, metadata: dict) -> bool:
if 'hide_from_vcr' in metadata:
if "hide_from_vcr" in metadata:
return True

code = code.strip()
for blocklisted_pattern in BLOCKLIST_COMMANDS:
if blocklisted_pattern in code:
Expand Down Expand Up @@ -116,8 +116,9 @@ def add_vcr_to_notebook(

cell_id = cell.get("id", idx)
cassette_name = f"{cassette_prefix}_{cell_id}.msgpack.zlib"
cell.source = f"with custom_vcr.use_cassette('{cassette_name}', filter_headers=['x-api-key', 'authorization'], record_mode='once', serializer='advanced_compressed'):\n" + "\n".join(
f" {line}" for line in lines
cell.source = (
f"with custom_vcr.use_cassette('{cassette_name}', filter_headers=['x-api-key', 'authorization'], record_mode='once', serializer='advanced_compressed'):\n"
+ "\n".join(f" {line}" for line in lines)
)

# Add import statement
Expand All @@ -129,7 +130,7 @@ def add_vcr_to_notebook(
"import base64",
"import zlib",
"import os",
"os.environ.pop(\"LANGCHAIN_TRACING_V2\", None)",
'os.environ.pop("LANGCHAIN_TRACING_V2", None)',
"custom_vcr = vcr.VCR()",
"",
"def compress_data(data, compression_level=9):",
Expand Down Expand Up @@ -188,13 +189,15 @@ def process_notebooks(should_comment_install_cells: bool) -> None:

# Add a special tag to the first code cell
if notebook.cells and notebook.cells[1].cell_type == "code":
notebook.cells[1].metadata["tags"] = notebook.cells[1].metadata.get("tags", []) + ["no_execution"]
notebook.cells[1].metadata["tags"] = notebook.cells[
1
].metadata.get("tags", []) + ["no_execution"]

nbformat.write(notebook, notebook_path)
logger.info(f"Processed: {notebook_path}")
except Exception as e:
logger.error(f"Error processing {notebook_path}: {e}")

with open(os.path.join(DOCS_PATH, "notebooks_no_execution.json"), "w") as f:
json.dump(NOTEBOOKS_NO_EXECUTION, f)

Expand Down
Loading