优化CommonDictionary的加载速度

hankcs · Nov 17, 2017 · 1969ebd · 1969ebd
1 parent e50be03
commit 1969ebd
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 1 deletion.
diff --git a/src/main/java/com/hankcs/hanlp/collection/trie/DoubleArrayTrie.java b/src/main/java/com/hankcs/hanlp/collection/trie/DoubleArrayTrie.java
@@ -535,6 +535,32 @@ public boolean load(ByteArray byteArray, V[] value)
         return true;
     }
 
+    /**
+     * 从字节数组加载（发现在MacOS上，此方法比ByteArray更快）
+     * @param bytes
+     * @param offset
+     * @param value
+     * @return
+     */
+    public boolean load(byte[] bytes, int offset, V[] value)
+    {
+        if (bytes == null) return false;
+        size = ByteUtil.bytesHighFirstToInt(bytes, offset);
+        offset += 4;
+        base = new int[size + 65535];   // 多留一些，防止越界
+        check = new int[size + 65535];
+        for (int i = 0; i < size; i++)
+        {
+            base[i] = ByteUtil.bytesHighFirstToInt(bytes, offset);
+            offset += 4;
+            check[i] = ByteUtil.bytesHighFirstToInt(bytes, offset);
+            offset += 4;
+        }
+        v = value;
+        used = null;    // 无用的对象,释放掉
+        return true;
+    }
+
     /**
      * 载入双数组，但是不提供值，此时本trie相当于一个set
      *

diff --git a/src/main/java/com/hankcs/hanlp/dictionary/common/CommonDictionary.java b/src/main/java/com/hankcs/hanlp/dictionary/common/CommonDictionary.java
@@ -105,7 +105,7 @@ protected boolean loadDat(ByteArray byteArray)
         {
             return false;
         }
-        return trie.load(byteArray, valueArray);
+        return trie.load(byteArray.getBytes(), byteArray.getOffset(), valueArray);
     }
 
     /**