diff --git a/test/conftest.py b/test/conftest.py
index 4ad832f42b..85910834b7 100644
--- a/test/conftest.py
+++ b/test/conftest.py
@@ -1,5 +1,6 @@
 from datetime import datetime
 from pathlib import Path
+from test.tracing.utils import SpyingTracer
 from typing import Generator
 from unittest.mock import Mock, patch
 
@@ -9,7 +10,6 @@
 
 from haystack import tracing
 from haystack.testing.test_utils import set_all_seeds
-from test.tracing.utils import SpyingTracer
 
 set_all_seeds(0)
 
@@ -82,3 +82,11 @@ def spying_tracer() -> Generator[SpyingTracer, None, None]:
 
     # Make sure to disable tracing after the test to avoid affecting other tests
     tracing.disable_tracing()
+
+
+# Collecting this test is causing issues when running tests in CI
+# as it's indirectly importing jsonschema which for some reason
+# is causing collection to fail.
+# See this issue:
+# https://github.com/python-jsonschema/jsonschema/issues/1236
+collect_ignore = ["components/validators/test_json_schema.py"]
diff --git a/test/core/pipeline/test_pipeline.py b/test/core/pipeline/test_pipeline.py
index 5e152300df..18bfe63b04 100644
--- a/test/core/pipeline/test_pipeline.py
+++ b/test/core/pipeline/test_pipeline.py
@@ -684,8 +684,9 @@ def test_describe_no_outputs():
 
 def test_from_template(monkeypatch):
     monkeypatch.setenv("OPENAI_API_KEY", "fake_key")
-    pipe = Pipeline.from_template(PredefinedPipeline.INDEXING)
-    assert pipe.get_component("cleaner")
+    with patch("haystack_integrations.document_stores.chroma.document_store.ChromaDocumentStore"):
+        pipe = Pipeline.from_template(PredefinedPipeline.INDEXING)
+        assert pipe.get_component("cleaner")
 
 
 def test_walk_pipeline_with_no_cycles():
diff --git a/test/core/pipeline/test_templates.py b/test/core/pipeline/test_templates.py
index 4f9e6a402d..ca87d48c85 100644
--- a/test/core/pipeline/test_templates.py
+++ b/test/core/pipeline/test_templates.py
@@ -1,4 +1,5 @@
 import tempfile
+from unittest import mock
 
 import pytest
 
@@ -46,14 +47,15 @@ def test_from_predefined(self):
     #  Building a pipeline directly using all default components specified in a predefined or custom template.
     def test_build_pipeline_with_default_components(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "fake_key")
-        rendered = PipelineTemplate.from_predefined(PredefinedPipeline.INDEXING).render()
-        pipeline = Pipeline.loads(rendered)
-
-        # pipeline has components
-        assert pipeline.get_component("cleaner")
-        assert pipeline.get_component("writer")
-        assert pipeline.get_component("embedder")
-
-        # pipeline should have inputs and outputs
-        assert len(pipeline.inputs()) > 0
-        assert len(pipeline.outputs()) > 0
+        with mock.patch("haystack_integrations.document_stores.chroma.document_store.ChromaDocumentStore"):
+            rendered = PipelineTemplate.from_predefined(PredefinedPipeline.INDEXING).render()
+            pipeline = Pipeline.loads(rendered)
+
+            # pipeline has components
+            assert pipeline.get_component("cleaner")
+            assert pipeline.get_component("writer")
+            assert pipeline.get_component("embedder")
+
+            # pipeline should have inputs and outputs
+            assert len(pipeline.inputs()) > 0
+            assert len(pipeline.outputs()) > 0