Use encoding_rs to add more character encoding support, including Chinese character encoding, such as GBK

Cargo.toml

@@ -18,6 +18,8 @@ yore = { version = "1.0.1", optional = true }
 datafusion = { version = "31", optional = true }
 datafusion-expr = { version = "31", optional = true }
 async-trait = { version = "0.1", optional = true }
+codepage= "0.1.2"
+encoding_rs = "0.8.35"
 
 [dev-dependencies]
 serde_derive = "1.0.102"

src/encoding.rs

@@ -167,6 +167,55 @@ impl Encoding for DynEncoding {
     }
 }
 
+
+pub struct EncodingRs(&'static  encoding_rs::Encoding);
+impl From<&'static encoding_rs::Encoding> for EncodingRs {
+    fn from(item: &'static encoding_rs::Encoding) -> Self {
+        EncodingRs(item)
+    }
+}
+impl Clone for EncodingRs  {
+    fn clone(&self) -> Self {
+        EncodingRs(self.0.clone())
+    }
+}
+
+impl AsCodePageMark for EncodingRs  {
+    fn code_page_mark(&self) -> crate::CodePageMark {
+        let code_page = codepage::from_encoding(self.0).unwrap();
+        match code_page {
+            1252 => crate::CodePageMark::CP1252,
+            866 => crate::CodePageMark::CP866,
+            874 => crate::CodePageMark::CP874,
+            1255 => crate::CodePageMark::CP1255,
+            1256 => crate::CodePageMark::CP1256,
+            1250 => crate::CodePageMark::CP1250,
+            1251 => crate::CodePageMark::CP1251,
+            1254 => crate::CodePageMark::CP1254,
+            1253 => crate::CodePageMark::CP1253,
+            65001 => crate::CodePageMark::Utf8,
+            950 => crate::CodePageMark::CP950,
+            949 => crate::CodePageMark::CP949,
+            936 => crate::CodePageMark::CP936,
+            932 => crate::CodePageMark::CP932,
+            _=> crate::CodePageMark::Utf8,
+        }
+
+    }
+}
+
+impl Encoding for  EncodingRs
+{
+    fn decode<'a>(&self, bytes: &'a [u8]) -> Result<Cow<'a, str>, DecodeError> {
+        Ok(self.0.decode(bytes).0)
+    }
+
+    fn encode<'a>(&self, s: &'a str) -> Result<Cow<'a, [u8]>, EncodeError> {
+        Ok(self.0.encode(s).0)
+    }
+}
+
+
 #[cfg(feature = "yore")]
 impl<T> Encoding for T
 where

src/header.rs

@@ -1,6 +1,6 @@
 use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
 
-use crate::encoding::DynEncoding;
+use crate::encoding::{DynEncoding, EncodingRs};
 use std::io::{Read, Write};
 
 use crate::field::types::Date;
@@ -94,7 +94,36 @@ impl CodePageMark {
         }
         #[cfg(not(feature = "yore"))]
         {
-            Some(DynEncoding::new(crate::encoding::UnicodeLossy))
+            //Some(DynEncoding::new(crate::encoding::UnicodeLossy))
+           let code_page = match self {
+                //CodePageMark::CP437 => 437,
+                //CodePageMark::CP850 => 850,
+                CodePageMark::CP1252 => 1252,
+                //CodePageMark::CP852 => 852 ,
+                CodePageMark::CP866 => 866 ,
+                //CodePageMark::CP865 => 865 ,
+                //CodePageMark::CP861 => 861,
+                CodePageMark::CP874 => 874,
+                CodePageMark::CP1255 => 1255,
+                CodePageMark::CP1256 => 1256,
+                CodePageMark::CP1250 => 1250,
+                CodePageMark::CP1251 => 1251,
+                CodePageMark::CP1254 => 1254,
+                CodePageMark::CP1253 => 1253,
+                CodePageMark::Utf8 => 65001,
+                //CodePageMark::CP895 => 895,
+                //CodePageMark::CP620 => 620,
+                //CodePageMark::CP737 => 737,
+                //CodePageMark::CP857 => 857,
+                CodePageMark::CP950 => 950,
+                CodePageMark::CP949 => 949,
+                CodePageMark::CP936 => 936,
+                CodePageMark::CP932 => 932,
+                _ => 65001
+            };
+            let gbk :EncodingRs = (codepage::to_encoding(code_page).unwrap()).into();
+            Some(DynEncoding::new(gbk))
+
         }
     }
 }
@@ -106,6 +135,7 @@ impl From<u8> for CodePageMark {
             0x01 => Self::CP437,
             0x02 => Self::CP850,
             0x03 => Self::CP1252,
+            0x4D => Self::CP936,
             // 0x04 => Self::StandardMacIntosh,
             0x64 => Self::CP852,
             0x65 => Self::CP866,

tests/tests.rs

@@ -18,6 +18,9 @@ const STATIONS_WITH_DELETED: &str = "./tests/data/stations_with_deleted.dbf";
 #[cfg(feature = "yore")]
 const CP850_DBF: &str = "tests/data/cp850.dbf";
 
+
+const CP936_DBF: &str = "tests/data/cp936.dbf";
+
 fn write_read_compare<R>(records: &Vec<R>, writer_builder: TableWriterBuilder)
 where
     R: WritableRecord + ReadableRecord + Debug + PartialEq,
@@ -361,3 +364,16 @@ fn test_record_marked_as_deleted_are_skipped_by_reader() -> Result<(), Box<dyn s
 
     Ok(())
 }
+
+
+#[test]
+fn test_codepages_cp936() {
+    let mut reader =
+        dbase::Reader::from_path(CP936_DBF).unwrap();
+    let records = reader.read().unwrap();
+
+    assert_eq!(
+        records[0].get("TEST"),
+        Some(&dbase::FieldValue::Character(Some("测试中文".to_string())))
+    ); 
+}