Add Expectations Draft

.github/actions/python-integration-tests/action.yml

@@ -33,7 +33,7 @@ runs:
     - name: Install dependencies
       run: |
         poetry install --with dev
-        poetry run pip install -U langchain langchain_anthropic tiktoken rapidfuzz vcrpy
+        poetry run pip install -U langchain langchain_anthropic tiktoken rapidfuzz vcrpy numpy
       shell: bash
       working-directory: python
 
@@ -52,7 +52,19 @@ runs:
         LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
         OPENAI_API_KEY: ${{ inputs.openai-api-key }}
         ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
-        LANGCHAIN_TEST_CACHE: "tests/cassettes"
       run: make doctest
       shell: bash
       working-directory: python
+
+
+    - name: Run Evaluation
+      env:
+          LANGCHAIN_TRACING_V2: "true"
+          LANGCHAIN_API_KEY: ${{ inputs.langchain-api-key }}
+          OPENAI_API_KEY: ${{ inputs.openai-api-key }}
+          ANTHROPIC_API_KEY: ${{ inputs.anthropic-api-key }}
+          LANGCHAIN_TEST_CACHE: "tests/cassettes"
+      run: make evals
+      shell: bash
+      working-directory: python
+

python/Makefile

@@ -1,4 +1,4 @@
-.PHONY: tests lint format
+.PHONY: tests lint format build publish doctest integration_tests integration_tests_fast evals
 
 tests:
 	poetry run pytest -n auto --durations=10 tests/unit_tests
@@ -15,6 +15,9 @@ integration_tests_fast:
 doctest:
 	poetry run pytest -n auto --durations=10 --doctest-modules langsmith
 
+evals:
+	poetry run pytest tests/evaluation
+
 lint:
 	poetry run ruff check .
 	poetry run mypy .

python/langsmith/__init__.py

@@ -3,8 +3,10 @@
 from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
+    from langsmith._expect import expect
     from langsmith._testing import unit
     from langsmith.client import Client
+    from langsmith.evaluation import aevaluate, evaluate
     from langsmith.evaluation.evaluator import EvaluationResult, RunEvaluator
     from langsmith.run_helpers import trace, traceable
     from langsmith.run_trees import RunTree
@@ -47,6 +49,19 @@ def __getattr__(name: str) -> Any:
 
         return unit
 
+    elif name == "expect":
+        from langsmith._expect import expect
+
+        return expect
+    elif name == "evaluate":
+        from langsmith.evaluation import evaluate
+
+        return evaluate
+    elif name == "aevaluate":
+        from langsmith.evaluation import aevaluate
+
+        return aevaluate
+
     raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
 
 
@@ -59,4 +74,7 @@ def __getattr__(name: str) -> Any:
     "traceable",
     "trace",
     "unit",
+    "expect",
+    "evaluate",
+    "aevaluate",
 ]