From 971dad7b34d58e3d2d55ee3fb17e5891693a4630 Mon Sep 17 00:00:00 2001 From: Kyle Verhoog Date: Fri, 22 Nov 2024 13:59:36 -0500 Subject: [PATCH] chore(langchain): disable flaky tests There appear to be stability issues with using snapshots and/or LangChain in general. There are failures in the mocked tests that look like: ``` builtins.AssertionError: assert 0 == 1 + where 0 = .call_count + where = .enqueue ``` as well as failures with snapshot based tests: ``` builtins.Failed: At request : At snapshot (token='tests.contrib.langchain.test_langchain_community.test_lcel_chain_simple'): - Directory: /go/src/github.com/DataDog/apm-reliability/dd-trace-py/tests/snapshots - CI mode: 1 - Trace File: /go/src/github.com/DataDog/apm-reliability/dd-trace-py/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_lcel_chain_simple.json - Stats File: /go/src/github.com/DataDog/apm-reliability/dd-trace-py/tests/snapshots/tests.contrib.langchain.test_langchain_community.test_lcel_chain_simple_tracestats.json At compare of 1 expected trace(s) to 0 received trace(s): Did not receive expected traces: 'langchain.request' ``` While we investigate a more stable method of testing it makes sense to disable the tests to avoid noise to our neighbours in the library :). DOWN WITH FLAKY TESTS --- tests/contrib/langchain/test_langchain_community.py | 2 ++ tests/contrib/langchain/test_langchain_llmobs.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/tests/contrib/langchain/test_langchain_community.py b/tests/contrib/langchain/test_langchain_community.py index 8fa415ec63..3066c6a0fb 100644 --- a/tests/contrib/langchain/test_langchain_community.py +++ b/tests/contrib/langchain/test_langchain_community.py @@ -36,6 +36,7 @@ def request_vcr(): yield get_request_vcr(subdirectory_name="langchain_community") +''' @pytest.mark.parametrize("ddtrace_config_langchain", [dict(logs_enabled=True, log_prompt_completion_sample_rate=1.0)]) def test_global_tags(ddtrace_config_langchain, langchain_openai, request_vcr, mock_metrics, mock_logs, mock_tracer): """ @@ -1535,3 +1536,4 @@ def circumference_tool(radius: float) -> float: ) calculator.invoke("2", config={"unserializable": object()}) +''' diff --git a/tests/contrib/langchain/test_langchain_llmobs.py b/tests/contrib/langchain/test_langchain_llmobs.py index ca8fe35883..c177f12dcd 100644 --- a/tests/contrib/langchain/test_langchain_llmobs.py +++ b/tests/contrib/langchain/test_langchain_llmobs.py @@ -198,6 +198,7 @@ def _invoke_tool(cls, tool, tool_input, config, mock_tracer): return mock_tracer.pop_traces()[0][0] +''' @pytest.mark.skipif(LANGCHAIN_VERSION >= (0, 1), reason="These tests are for langchain < 0.1.0") class TestLLMObsLangchain(BaseTestLLMObsLangchain): cassette_subdirectory_name = "langchain" @@ -1185,3 +1186,4 @@ def test_llmobs_with_anthropic_disabled(self): self._call_anthropic_chat(ChatAnthropic) self._assert_trace_structure_from_writer_call_args(["llm"]) +'''