diff --git a/jcvi/graphics/karyotype.py b/jcvi/graphics/karyotype.py
index 1cbaa273..0ce97d43 100644
--- a/jcvi/graphics/karyotype.py
+++ b/jcvi/graphics/karyotype.py
@@ -359,9 +359,9 @@ def __init__(
         fp = open(seqidsfile)
         # Strip the reverse orientation tag for e.g. chr3-
         di = lambda x: x[:-1] if x[-1] == "-" else x
-        for i, row in enumerate(fp):
-            if row[0] == "#":
-                continue
+        # Comments can cause layout and seqids to be out of sync
+        # https://github.com/tanghaibao/jcvi/issues/676
+        for i, row in enumerate(_ for _ in fp if not _.startswith("#")):
             t = layout[i]
             # There can be comments in seqids file:
             # https://github.com/tanghaibao/jcvi/issues/335
diff --git a/tests/graphics/data/seqids_with_comments b/tests/graphics/data/seqids_with_comments
new file mode 100644
index 00000000..f1bb0a5a
--- /dev/null
+++ b/tests/graphics/data/seqids_with_comments
@@ -0,0 +1,6 @@
+#aa
+chr1,chr2,chr3,chr4,chr5,chr6,chr7,chr8,chr9,chr10,chr11,chr12,chr13,chr14,chr15,chr16,chr17,chr18,chr19
+#bb
+#cc
+Pp01,Pp02,Pp03,Pp04,Pp05,Pp06,Pp07,Pp08
+#dd
diff --git a/tests/graphics/test_karyotype.py b/tests/graphics/test_karyotype.py
index 65240f48..78b5a6cb 100644
--- a/tests/graphics/test_karyotype.py
+++ b/tests/graphics/test_karyotype.py
@@ -31,4 +31,9 @@ def test_main():
     cleanup("karyotype.pdf")
     image_name = karyotype_main(["seqids", "layout"])
     assert op.exists(image_name)
+    cleanup("karyotype_with_comments.pdf")
+    image_name = karyotype_main(
+        ["seqids_with_comments", "layout", "-o", "karyotype_with_comments.pdf"]
+    )
+    assert op.exists(image_name)
     os.chdir(cwd)