diff --git a/setup.py b/setup.py
index c0df285dcffb..13c93dcae3c0 100644
--- a/setup.py
+++ b/setup.py
@@ -95,7 +95,6 @@
     "Jinja2",
     "k-diffusion>=0.0.12",
     "librosa",
-    "note-seq",
     "numpy",
     "parameterized",
     "protobuf>=3.20.3,<4",
@@ -191,7 +190,6 @@ def run(self):
     "Jinja2",
     "k-diffusion",
     "librosa",
-    "note-seq",
     "parameterized",
     "pytest",
     "pytest-timeout",
diff --git a/src/diffusers/dependency_versions_table.py b/src/diffusers/dependency_versions_table.py
index 1269cf1578a6..0e714accacd6 100644
--- a/src/diffusers/dependency_versions_table.py
+++ b/src/diffusers/dependency_versions_table.py
@@ -19,7 +19,6 @@
     "Jinja2": "Jinja2",
     "k-diffusion": "k-diffusion>=0.0.12",
     "librosa": "librosa",
-    "note-seq": "note-seq",
     "numpy": "numpy",
     "parameterized": "parameterized",
     "protobuf": "protobuf>=3.20.3,<4",
diff --git a/tests/pipelines/spectrogram_diffusion/test_spectrogram_diffusion.py b/tests/pipelines/spectrogram_diffusion/test_spectrogram_diffusion.py
index 3b64ea2d2fc1..3ec6f681be79 100644
--- a/tests/pipelines/spectrogram_diffusion/test_spectrogram_diffusion.py
+++ b/tests/pipelines/spectrogram_diffusion/test_spectrogram_diffusion.py
@@ -34,6 +34,10 @@
 MIDI_FILE = "./tests/fixtures/elise_format0.mid"
 
 
+# The note-seq package throws an error on import because the default installed version of Ipython
+# is not compatible with python 3.8 which we run in the CI.
+# https://github.com/huggingface/diffusers/actions/runs/4830121056/jobs/8605954838#step:7:98
+@unittest.skip("The note-seq package currently throws an error on import")
 class SpectrogramDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase):
     pipeline_class = SpectrogramDiffusionPipeline
     required_optional_params = PipelineTesterMixin.required_optional_params - {