infra: individual template CI

.github/scripts/check_diff.py

@@ -24,18 +24,21 @@
             )
         ):
             dirs_to_run.update(LANGCHAIN_DIRS)
-        elif "libs/community" in file:
+        elif file.startswith("libs/community"):
             dirs_to_run.update(
                 ("libs/community", "libs/langchain", "libs/experimental")
             )
-        elif "libs/partners" in file:
+        elif file.startswith("libs/partners"):
             partner_dir = file.split("/")[2]
             dirs_to_run.update(
                 (f"libs/partners/{partner_dir}", "libs/langchain", "libs/experimental")
             )
-        elif "libs/langchain" in file:
+        elif file.startswith("templates/"):
+            template = file.split("/")[1]
+            dirs_to_run.add(f"templates/{template}")
+        elif file.startswith("libs/langchain"):
             dirs_to_run.update(("libs/langchain", "libs/experimental"))
-        elif "libs/experimental" in file:
+        elif file.startswith("libs/experimental"):
             dirs_to_run.add("libs/experimental")
         elif file.startswith("libs/"):
             dirs_to_run.update(LANGCHAIN_DIRS)

.github/workflows/_all_ci.yml

@@ -42,24 +42,28 @@ jobs:
     secrets: inherit
 
   test:
+    if: ${{ ! startsWith(inputs.working-directory, 'templates/') }}
     uses: ./.github/workflows/_test.yml
     with:
       working-directory: ${{ inputs.working-directory }}
     secrets: inherit
 
   compile-integration-tests:
+    if: ${{ ! startsWith(inputs.working-directory, 'templates/') }}
     uses: ./.github/workflows/_compile_integration_test.yml
     with:
       working-directory: ${{ inputs.working-directory }}
     secrets: inherit
 
   dependencies:
+    if: ${{ ! startsWith(inputs.working-directory, 'templates/') }}
     uses: ./.github/workflows/_dependencies.yml
     with:
       working-directory: ${{ inputs.working-directory }}
     secrets: inherit
 
   extended-tests:
+    if: ${{ ! startsWith(inputs.working-directory, 'libs/partners/') && ! startsWith(inputs.working-directory, 'templates/') }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -72,7 +76,6 @@ jobs:
     defaults:
       run:
         working-directory: ${{ inputs.working-directory }}
-    if: ${{ ! startsWith(inputs.working-directory, 'libs/partners/') }}
     steps:
       - uses: actions/checkout@v4
 

libs/cli/langchain_cli/namespaces/template.py

@@ -9,12 +9,22 @@
 from typing import Optional
 
 import typer
-from typing_extensions import Annotated
+from typing_extensions import Annotated, TypedDict
 
+from langchain_cli.utils.find_replace import replace_glob
 from langchain_cli.utils.packages import get_langserve_export, get_package_root
 
 package_cli = typer.Typer(no_args_is_help=True, add_completion=False)
 
+Replacements = TypedDict(
+    "Replacements",
+    {
+        "__package_name__": str,
+        "__module_name__": str,
+        "__app_route_code__": str,
+    },
+)
+
 
 @package_cli.command()
 def new(
@@ -53,33 +63,20 @@ def new(
         f'add_routes(app, {chain_name}, path="/{package_name}")'
     )
 
-    # replace template strings
-    pyproject = destination_dir / "pyproject.toml"
-    pyproject_contents = pyproject.read_text()
-    pyproject.write_text(
-        pyproject_contents.replace("__package_name__", package_name).replace(
-            "__module_name__", module_name
-        )
-    )
-
     # move module folder
     package_dir = destination_dir / module_name
     shutil.move(destination_dir / "package_template", package_dir)
 
-    # update init
-    init = package_dir / "__init__.py"
-    init_contents = init.read_text()
-    init.write_text(init_contents.replace("__module_name__", module_name))
-
-    # replace readme
-    readme = destination_dir / "README.md"
-    readme_contents = readme.read_text()
-    readme.write_text(
-        readme_contents.replace("__package_name__", package_name).replace(
-            "__app_route_code__", app_route_code
-        )
+    replacements = Replacements(
+        {
+            "__package_name__": package_name,
+            "__module_name__": module_name,
+            "__app_route_code__": app_route_code,
+        }
     )
 
+    replace_glob(destination_dir, "**/*", replacements)
+
     # poetry install
     if with_poetry:
         subprocess.run(["poetry", "install"], cwd=destination_dir)

libs/cli/langchain_cli/package_template/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all format lint test tests help
+
+# Default target executed when no arguments are given to make.
+all: help
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/
+
+test:
+	poetry run pytest $(TEST_FILE)
+
+PYTHON_FILES=.
+MYPY_CACHE=.mypy_cache
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=templates/__package_name__ --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=__module_name__
+lint_tests: PYTHON_FILES=tests
+lint_tests: MYPY_CACHE=.mypy_cache_test
+
+lint lint_diff lint_package lint_tests:
+	poetry run ruff .
+	poetry run ruff format $(PYTHON_FILES) --diff
+	poetry run ruff --select I $(PYTHON_FILES)
+	poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
+
+format format_diff:
+	poetry run ruff format $(PYTHON_FILES)
+	poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'check_imports				- check imports'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'test                         - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'

libs/cli/langchain_cli/package_template/pyproject.toml

@@ -7,11 +7,46 @@ readme = "README.md"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<4.0"
-langchain = ">=0.0.313, <0.1"
-openai = "^0.28.1"
+langchain = ">=0.0.350, <0.2"
+
+[tool.poetry.group.test]
+optional = true
+
+[tool.poetry.group.test.dependencies]
+pytest = "^7.3.0"
+freezegun = "^1.2.2"
+pytest-mock  = "^3.10.0"
+syrupy = "^4.0.2"
+pytest-watcher = "^0.3.4"
+pytest-asyncio = "^0.21.1"
+langchain-core = {path = "../../libs/core", develop = true}
+
+[tool.poetry.group.codespell]
+optional = true
+
+[tool.poetry.group.codespell.dependencies]
+codespell = "^2.2.6"
+
+[tool.poetry.group.test_integration]
+optional = true
+
+[tool.poetry.group.test_integration.dependencies]
+
+[tool.poetry.group.lint]
+optional = true
+
+[tool.poetry.group.lint.dependencies]
+ruff = "^0.1.8"
+
+[tool.poetry.group.typing.dependencies]
+mypy = "^1.7.1"
+langchain-core = {path = "../../libs/core", develop = true}
+
+[tool.poetry.group.dev]
+optional = true
 
 [tool.poetry.group.dev.dependencies]
-langchain-cli = ">=0.0.4"
+langchain-cli = ">=0.0.19"
 fastapi = "^0.104.0"
 sse-starlette = "^1.6.5"
 

libs/cli/langchain_cli/utils/git.py

@@ -191,7 +191,6 @@ def copy_repo(
     Raises FileNotFound error if it can't find source
     """
 
-    def ignore_func(_, files):
-        return [f for f in files if f == ".git"]
-
-    shutil.copytree(source, destination, ignore=ignore_func)
+    shutil.copytree(
+        source, destination, ignore=shutil.ignore_patterns(".git", "Makefile")
+    )

templates/pirate-speak/Makefile

@@ -0,0 +1,46 @@
+.PHONY: all format lint test tests integration_tests docker_tests help extended_tests
+
+# Default target executed when no arguments are given to make.
+all: help
+
+# Define a variable for the test file path.
+TEST_FILE ?= tests/
+
+test:
+	poetry run pytest $(TEST_FILE)
+
+PYTHON_FILES=.
+MYPY_CACHE=.mypy_cache
+lint format: PYTHON_FILES=.
+lint_diff format_diff: PYTHON_FILES=$(shell git diff --relative=templates/pirate-speak --name-only --diff-filter=d master | grep -E '\.py$$|\.ipynb$$')
+lint_package: PYTHON_FILES=pirate_speak
+lint_tests: PYTHON_FILES=tests
+lint_tests: MYPY_CACHE=.mypy_cache_test
+
+lint lint_diff lint_package lint_tests:
+	poetry run ruff .
+	poetry run ruff format $(PYTHON_FILES) --diff
+	poetry run ruff --select I $(PYTHON_FILES)
+	poetry run mypy $(PYTHON_FILES) --cache-dir $(MYPY_CACHE)
+
+format format_diff:
+	poetry run ruff format $(PYTHON_FILES)
+	poetry run ruff --select I --fix $(PYTHON_FILES)
+
+spell_check:
+	poetry run codespell --toml pyproject.toml
+
+spell_fix:
+	poetry run codespell --toml pyproject.toml -w
+
+######################
+# HELP
+######################
+
+help:
+	@echo '----'
+	@echo 'check_imports				- check imports'
+	@echo 'format                       - run code formatters'
+	@echo 'lint                         - run linters'
+	@echo 'test                         - run unit tests'
+	@echo 'test TEST_FILE=<test_file>   - run all tests in file'