From ce8ec9e05d1a990e5bd229da07162d6d7df71304 Mon Sep 17 00:00:00 2001
From: William FH <13333726+hinthornw@users.noreply.github.com>
Date: Wed, 21 Aug 2024 17:30:18 -0700
Subject: [PATCH] [Py] Handle experiment name conflicts (#940)

---
 python/langsmith/evaluation/_runner.py        | 40 +++++++++++--------
 .../test_experiment_manager.py                | 25 ++++++++++++
 2 files changed, 49 insertions(+), 16 deletions(-)
 create mode 100644 python/tests/integration_tests/test_experiment_manager.py

diff --git a/python/langsmith/evaluation/_runner.py b/python/langsmith/evaluation/_runner.py
index f85dcc482..d1870e989 100644
--- a/python/langsmith/evaluation/_runner.py
+++ b/python/langsmith/evaluation/_runner.py
@@ -30,7 +30,6 @@
     cast,
 )
 
-from requests import HTTPError
 from typing_extensions import TypedDict
 
 import langsmith
@@ -958,24 +957,33 @@ def _get_experiment_metadata(self):
             }
         return project_metadata
 
-    def _get_project(self, first_example: schemas.Example) -> schemas.TracerSession:
-        if self._experiment is None:
+    def _create_experiment(
+        self, dataset_id: uuid.UUID, metadata: dict
+    ) -> schemas.TracerSession:
+        # There is a chance of name collision, so we'll retry
+        starting_name = self._experiment_name
+        num_attempts = 10
+        for _ in range(num_attempts):
             try:
-                project_metadata = self._get_experiment_metadata()
-                project = self.client.create_project(
-                    self.experiment_name,
+                return self.client.create_project(
+                    self._experiment_name,
                     description=self._description,
-                    reference_dataset_id=first_example.dataset_id,
-                    metadata=project_metadata,
-                )
-            except (HTTPError, ValueError, ls_utils.LangSmithError) as e:
-                if "already exists " not in str(e):
-                    raise e
-                raise ValueError(
-                    # TODO: Better error
-                    f"Experiment {self.experiment_name} already exists."
-                    " Please use a different name."
+                    reference_dataset_id=dataset_id,
+                    metadata=metadata,
                 )
+            except ls_utils.LangSmithConflictError:
+                self._experiment_name = f"{starting_name}-{str(uuid.uuid4().hex[:6])}"
+        raise ValueError(
+            f"Could not find a unique experiment name in {num_attempts} attempts."
+            " Please try again with a different experiment name."
+        )
+
+    def _get_project(self, first_example: schemas.Example) -> schemas.TracerSession:
+        if self._experiment is None:
+            project_metadata = self._get_experiment_metadata()
+            project = self._create_experiment(
+                first_example.dataset_id, project_metadata
+            )
         else:
             project = self._experiment
         return project
diff --git a/python/tests/integration_tests/test_experiment_manager.py b/python/tests/integration_tests/test_experiment_manager.py
new file mode 100644
index 000000000..93f35a709
--- /dev/null
+++ b/python/tests/integration_tests/test_experiment_manager.py
@@ -0,0 +1,25 @@
+import uuid
+
+from langsmith.client import Client
+from langsmith.evaluation._runner import _ExperimentManager
+
+
+def test_experiment_manager_existing_name():
+    client = Client()
+    dataset_name = f"Test Dups: {str(uuid.uuid4())}"
+    ds = client.create_dataset(dataset_name)
+    client.create_example(inputs={"un": "important"}, dataset_id=ds.id)
+    prefix = "Some Test Prefix"
+    try:
+        manager = _ExperimentManager(dataset_name, experiment=prefix, client=client)
+        assert manager is not None
+        original_name = manager._experiment_name
+        assert original_name.startswith(prefix)
+        client.create_project(original_name, reference_dataset_id=ds.id)
+        manager.start()
+        new_name = manager._experiment_name
+        assert new_name.startswith(prefix)
+        assert new_name != original_name
+
+    finally:
+        client.delete_dataset(dataset_id=ds.id)