From 1eb79315ae432a9365321f10b34ca0e1e2dce21e Mon Sep 17 00:00:00 2001 From: Bingfeng Xia Date: Fri, 3 Apr 2020 20:56:13 -0700 Subject: [PATCH] Implement WeakIdentityHashMap in avro-fastserde --- avro-fastserde/build.gradle | 1 - .../fastserde/FastDatumReaderWriterUtil.java | 6 +- .../avro/fastserde/WeakIdentityHashMap.java | 197 ++++++++++++++++++ 3 files changed, 199 insertions(+), 5 deletions(-) create mode 100644 avro-fastserde/src/main/java/com/linkedin/avro/fastserde/WeakIdentityHashMap.java diff --git a/avro-fastserde/build.gradle b/avro-fastserde/build.gradle index e20f7df27..e6bc1ca72 100644 --- a/avro-fastserde/build.gradle +++ b/avro-fastserde/build.gradle @@ -53,7 +53,6 @@ dependencies { compile "org.apache.commons:commons-lang3:3.4" compile "com.sun.codemodel:codemodel:2.6" compile "com.google.guava:guava:19.0" - compile "org.jboss:jboss-common-core:2.5.0.Final" // By default, the compile and testCompile configuration is using avro-1.8, and // if you need to switch to an old version of Avro, we need to make diff --git a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastDatumReaderWriterUtil.java b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastDatumReaderWriterUtil.java index 4c3f844e2..95558ea54 100644 --- a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastDatumReaderWriterUtil.java +++ b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/FastDatumReaderWriterUtil.java @@ -2,10 +2,8 @@ import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.avro.Schema; -import org.jboss.util.collection.WeakIdentityHashMap; /** * Utility class to generate and reuse FastDatumReader/Writer. The cache key is schema object. @@ -49,10 +47,10 @@ public int hashCode() { //TODO : LRU cache private static final Map> fastGenericDatumReaderCache = new FastAvroConcurrentHashMap<>(); - private static final Map> fastGenericDatumWriterCache = new WeakIdentityHashMap(); + private static final Map> fastGenericDatumWriterCache = new WeakIdentityHashMap<>(); private static final Map> fastSpecificDatumReaderCache = new FastAvroConcurrentHashMap<>(); - private static final Map> fastSpecificDatumWriterCache = new WeakIdentityHashMap(); + private static final Map> fastSpecificDatumWriterCache = new WeakIdentityHashMap<>(); private FastDatumReaderWriterUtil() { } diff --git a/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/WeakIdentityHashMap.java b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/WeakIdentityHashMap.java new file mode 100644 index 000000000..57de5a957 --- /dev/null +++ b/avro-fastserde/src/main/java/com/linkedin/avro/fastserde/WeakIdentityHashMap.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package com.linkedin.avro.fastserde; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Implements a combination of WeakHashMap and IdentityHashMap. Useful for + * caches that need to key off of a == comparison instead of a .equals. + * + * This class is not a general-purpose Map implementation! While this class + * implements the Map interface, it intentionally violates Map's general + * contract, which mandates the use of the equals method when comparing objects. + * This class is designed for use only in the rare cases wherein + * reference-equality semantics are required. + * + * Note that this implementation is not synchronized. + */ +public class WeakIdentityHashMap implements Map { + private final ReferenceQueue queue = new ReferenceQueue<>(); + private Map backingStore = new HashMap<>(); + + public WeakIdentityHashMap() { + } + + @Override + public void clear() { + backingStore.clear(); + reap(); + } + + @Override + public boolean containsKey(Object key) { + reap(); + return backingStore.containsKey(new IdentityWeakReference(key)); + } + + @Override + public boolean containsValue(Object value) { + reap(); + return backingStore.containsValue(value); + } + + @Override + public Set> entrySet() { + reap(); + Set> ret = new HashSet<>(); + for (Map.Entry ref : backingStore.entrySet()) { + final K key = ref.getKey().get(); + final V value = ref.getValue(); + Map.Entry entry = new Map.Entry() { + @Override + public K getKey() { + return key; + } + + @Override + public V getValue() { + return value; + } + + @Override + public V setValue(V value) { + throw new UnsupportedOperationException(); + } + }; + ret.add(entry); + } + return Collections.unmodifiableSet(ret); + } + + @Override + public Set keySet() { + reap(); + Set ret = new HashSet<>(); + for (IdentityWeakReference ref : backingStore.keySet()) { + ret.add(ref.get()); + } + return Collections.unmodifiableSet(ret); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof WeakIdentityHashMap)) { + return false; + } + return backingStore.equals(((WeakIdentityHashMap) o).backingStore); + } + + @Override + public V get(Object key) { + reap(); + return backingStore.get(new IdentityWeakReference(key)); + } + + @Override + public V put(K key, V value) { + reap(); + return backingStore.put(new IdentityWeakReference(key), value); + } + + @Override + public int hashCode() { + reap(); + return backingStore.hashCode(); + } + + @Override + public boolean isEmpty() { + reap(); + return backingStore.isEmpty(); + } + + @Override + public void putAll(Map t) { + throw new UnsupportedOperationException(); + } + + @Override + public V remove(Object key) { + reap(); + return backingStore.remove(new IdentityWeakReference(key)); + } + + @Override + public int size() { + reap(); + return backingStore.size(); + } + + @Override + public Collection values() { + reap(); + return backingStore.values(); + } + + private synchronized void reap() { + Object zombie = queue.poll(); + + while (zombie != null) { + IdentityWeakReference victim = (IdentityWeakReference) zombie; + backingStore.remove(victim); + zombie = queue.poll(); + } + } + + class IdentityWeakReference extends WeakReference { + int hash; + + @SuppressWarnings("unchecked") + IdentityWeakReference(Object obj) { + super((K) obj, queue); + hash = System.identityHashCode(obj); + } + + @Override + public int hashCode() { + return hash; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof WeakIdentityHashMap.IdentityWeakReference)) { + return false; + } + IdentityWeakReference ref = (IdentityWeakReference) o; + return this.get() == ref.get(); + } + } +} \ No newline at end of file