wip new msgspec decoder

mmtfPyspark/io/mmtfReader.py

@@ -25,6 +25,10 @@
 import urllib
 import urllib.request as urllib2
 import pandas as pd
+# -pr
+# -pr import pandas as pd
+import msgspec
+# -pr
 
 text = "org.apache.hadoop.io.Text"
 byteWritable = "org.apache.hadoop.io.BytesWritable"
@@ -230,7 +234,10 @@ def _get_structure(pdbId, reduced, first_model):
             data = gzip.decompress(response.read())
         else:
             data = response.read()
-        unpack = pd.read_msgpack(data)
+        # -pr unpack = pd.read_msgpack(data)
+        dc_msg = msgspec.json.Decoder()
+        unpack = dc_msg.decode(data)
+        # -pr
         decoder = MmtfStructure(unpack, first_model)
         return (pdbId, decoder)
     except urllib.error.HTTPError:
@@ -245,7 +252,10 @@ def _call_sequence_file(t, first_model):
     # decoder = MmtfStructure(unpack)
     # return (str(t[0]), decoder)
     data = gzip.decompress(t[1])
-    unpack = pd.read_msgpack(data)
+    # -pr unpack = pd.read_msgpack(data)
+    dc_msg = msgspec.json.Decoder()
+    unpack = dc_msg.decode(data)
+    # -pr
     decoder = MmtfStructure(unpack, first_model)
     return (t[0], decoder)
 
@@ -257,7 +267,10 @@ def _call_mmtf(f, first_model=False):
         name = f.split('/')[-1].split('.')[0].upper()
         data = gzip.open(f, 'rb')
         #unpack = msgpack.unpack(data, raw=False)
-        unpack = pd.read_msgpack(data)
+        # -pr unpack = pd.read_msgpack(data)
+        dc_msg = msgspec.json.Decoder()
+        unpack = dc_msg.decode(data)
+        # -pr
         decoder = MmtfStructure(unpack, first_model)
         return (name, decoder)
 
@@ -266,7 +279,10 @@ def _call_mmtf(f, first_model=False):
         #unpack = msgpack.unpack(open(f, "rb"), raw=False)
         #decoder = MmtfStructure(unpack)
         name = f.split('/')[-1].split('.')[0].upper()
-        unpack = pd.read_msgpack(f)
+        # -prunpack = pd.read_msgpack(f)
+        dc_msg = msgspec.json.Decoder()
+        unpack = dc_msg.decode(f)
+        # -pr
         decoder = MmtfStructure(unpack, first_model)
         return (name, decoder)