Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Switch from trove to fastutil #54

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Switch from trove to fastutil #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
651239f
Switch from trove to fastutil
benmccann Aug 17, 2017
a9072b9
Merge branch 'fastutil' of https://github.com/benmccann/morpheus-core…
Zavster Aug 22, 2017
ab35ff4
Use non-deprecated methods in Fasutil and fix test errors
Zavster Aug 22, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@
*.iws
.idea/

# Eclipse files
.settings
.classpath
.project
bin/

# MC FILES
.DS_Store

Expand Down
6 changes: 3 additions & 3 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,9 @@
</dependency>

<dependency>
<groupId>net.sf.trove4j</groupId>
<artifactId>trove4j</artifactId>
<version>3.0.3</version>
<groupId>it.unimi.dsi</groupId>
<artifactId>fastutil</artifactId>
<version>7.2.1</version>
</dependency>

<dependency>
Expand Down
6 changes: 4 additions & 2 deletions src/main/java/com/zavtech/morpheus/array/ArrayBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@
import com.zavtech.morpheus.util.SortAlgorithm;
import com.zavtech.morpheus.util.functions.ToBooleanFunction;

import it.unimi.dsi.fastutil.Arrays;

/**
* A convenience base class used to build Morpheus Array implementations
*
Expand Down Expand Up @@ -184,7 +186,7 @@ public Array<T> fill(T value) {

@Override()
public Array<T> distinct() {
return distinct(Integer.MAX_VALUE);
return distinct(Arrays.MAX_ARRAY_SIZE);
}


Expand All @@ -196,7 +198,7 @@ public Array<T> distinct(int limit) {
case LONG: return (Array<T>)ArrayUtils.distinct(stream().longs(), limit);
case DOUBLE: return (Array<T>)ArrayUtils.distinct(stream().doubles(), limit);
default:
final int capacity = limit < Integer.MAX_VALUE ? limit : 16;
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final Set<T> set = new HashSet<>(capacity);
final ArrayBuilder<T> builder = ArrayBuilder.of(capacity, type());
for (int i=0; i<length(); ++i) {
Expand Down
63 changes: 34 additions & 29 deletions src/main/java/com/zavtech/morpheus/array/ArrayUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,16 @@
import java.util.stream.LongStream;
import java.util.stream.Stream;

import gnu.trove.set.TDoubleSet;
import gnu.trove.set.TIntSet;
import gnu.trove.set.TLongSet;
import gnu.trove.set.hash.TDoubleHashSet;
import gnu.trove.set.hash.TIntHashSet;
import gnu.trove.set.hash.TLongHashSet;

import com.zavtech.morpheus.index.Index;
import com.zavtech.morpheus.range.Range;

import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
import it.unimi.dsi.fastutil.doubles.DoubleSet;
import it.unimi.dsi.fastutil.ints.IntOpenHashSet;
import it.unimi.dsi.fastutil.ints.IntSet;
import it.unimi.dsi.fastutil.longs.LongOpenHashSet;
import it.unimi.dsi.fastutil.longs.LongSet;

/**
* A utility class that provides various useful functions to operate on Morpheus arrays
*
Expand Down Expand Up @@ -114,7 +114,8 @@ public static <V> Array<V> toArray(Iterable<V> values) {
* @return the array of distinct values in the order they were observed
*/
public static Array<Integer> distinct(IntStream values, int limit) {
final DistinctInts distinct = new DistinctInts(limit);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we throw a new IllegalArgumentException here if limit > MAX_ARRAY_SIZE? I'm wondering if it'll be confusing users if we use a value that's different than what they passed in

final DistinctInts distinct = new DistinctInts(limit, capacity);
final PrimitiveIterator.OfInt iterator = values.iterator();
while (iterator.hasNext()) {
final int value = iterator.next();
Expand All @@ -134,7 +135,8 @@ public static Array<Integer> distinct(IntStream values, int limit) {
* @return the array of distinct values in the order they were observed
*/
public static Array<Long> distinct(LongStream values, int limit) {
final DistinctLongs distinct = new DistinctLongs(limit);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final DistinctLongs distinct = new DistinctLongs(limit, capacity);
final PrimitiveIterator.OfLong iterator = values.iterator();
while (iterator.hasNext()) {
final long value = iterator.next();
Expand All @@ -154,7 +156,8 @@ public static Array<Long> distinct(LongStream values, int limit) {
* @return the array of distinct values in the order they were observed
*/
public static Array<Double> distinct(DoubleStream values, int limit) {
final DistinctDoubles distinct = new DistinctDoubles(limit);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final DistinctDoubles distinct = new DistinctDoubles(limit, capacity);
final PrimitiveIterator.OfDouble iterator = values.iterator();
while (iterator.hasNext()) {
final double value = iterator.next();
Expand All @@ -174,7 +177,8 @@ public static Array<Double> distinct(DoubleStream values, int limit) {
* @return the array of distinct values in the order they were observed
*/
public static <V> Array<V> distinct(Stream<V> values, int limit) {
final DistinctValues<V> distinct = new DistinctValues<>(limit);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final DistinctValues<V> distinct = new DistinctValues<>(limit, capacity);
final Iterator<V> iterator = values.iterator();
while (iterator.hasNext()) {
final V value = iterator.next();
Expand All @@ -198,10 +202,10 @@ private static class DistinctCalculator<T> {
/**
* Constructor
* @param type the data type
* @param limit the limit
* @param capacity the initial capacity for array builder
*/
DistinctCalculator(Class<T> type, int limit) {
this.builder = ArrayBuilder.of(limit < Integer.MAX_VALUE ? limit : 1000, type);
DistinctCalculator(Class<T> type, int capacity) {
this.builder = ArrayBuilder.of(capacity, type);
}

/**
Expand All @@ -219,16 +223,17 @@ public final Array<T> toArray() {
private static class DistinctInts extends DistinctCalculator<Integer> {

private int limit;
private TIntSet distinctSet;
private IntSet distinctSet;

/**
* Constructor
* @param limit the limit for this calculator
* @param capacity the initial capacity for set
*/
DistinctInts(int limit) {
super(Integer.class, limit);
DistinctInts(int limit, int capacity) {
super(Integer.class, capacity);
this.limit = limit;
this.distinctSet = new TIntHashSet(limit < Integer.MAX_VALUE ? limit : 1000);
this.distinctSet = new IntOpenHashSet(capacity);
}

/**
Expand All @@ -250,16 +255,16 @@ public final boolean add(int value) {
private static class DistinctLongs extends DistinctCalculator<Long> {

private int limit;
private TLongSet distinctSet;
private LongSet distinctSet;

/**
* Constructor
* @param limit the limit for this calculator
*/
DistinctLongs(int limit) {
super(Long.class, limit);
DistinctLongs(int limit, int capacity) {
super(Long.class, capacity);
this.limit = limit;
this.distinctSet = new TLongHashSet(limit < Integer.MAX_VALUE ? limit : 1000);
this.distinctSet = new LongOpenHashSet(capacity);
}

/**
Expand All @@ -280,16 +285,16 @@ public final boolean add(long value) {
private static class DistinctDoubles extends DistinctCalculator<Double> {

private int limit;
private TDoubleSet distinctSet;
private DoubleSet distinctSet;

/**
* Constructor
* @param limit the limit for this calculator
*/
DistinctDoubles(int limit) {
super(Double.class, limit);
DistinctDoubles(int limit, int capacity) {
super(Double.class, capacity);
this.limit = limit;
this.distinctSet = new TDoubleHashSet(limit < Integer.MAX_VALUE ? limit : 1000);
this.distinctSet = new DoubleOpenHashSet(capacity);
}

/**
Expand Down Expand Up @@ -317,10 +322,10 @@ private static class DistinctValues<V> extends DistinctCalculator<V> {
* @param limit the limit for this calculator
*/
@SuppressWarnings("unchecked")
DistinctValues(int limit) {
super((Class<V>)Object.class, limit);
DistinctValues(int limit, int capacity) {
super((Class<V>)Object.class, capacity);
this.limit = limit;
this.distinctSet = new HashSet<>(limit < Integer.MAX_VALUE ? limit : 1000);
this.distinctSet = new HashSet<>(capacity);
}

/**
Expand Down
27 changes: 15 additions & 12 deletions src/main/java/com/zavtech/morpheus/array/coding/IntCoding.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@
import java.util.TimeZone;
import java.util.stream.IntStream;

import gnu.trove.map.TObjectIntMap;
import gnu.trove.map.hash.TObjectIntHashMap;

import com.zavtech.morpheus.util.IntComparator;
import com.zavtech.morpheus.util.SortAlgorithm;

import it.unimi.dsi.fastutil.objects.Object2IntMap;
import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;

/**
* An interface that exposes a coding between object values and corresponding int code
*
Expand Down Expand Up @@ -169,15 +169,16 @@ class OfCurrency extends BaseCoding<Currency> implements IntCoding<Currency> {
private static final long serialVersionUID = 1L;

private final Currency[] currencies;
private final TObjectIntMap<Currency> codeMap;
private final Object2IntMap<Currency> codeMap;

/**
* Constructor
*/
public OfCurrency() {
super(Currency.class);
this.currencies = Currency.getAvailableCurrencies().stream().toArray(Currency[]::new);
this.codeMap = new TObjectIntHashMap<>(currencies.length, 0.5f, -1);
this.codeMap = new Object2IntOpenHashMap<>(currencies.length, 0.5f);
this.codeMap.defaultReturnValue(-1);
Arrays.sort(currencies, (c1, c2) -> c1.getCurrencyCode().compareTo(c2.getCurrencyCode()));
for (int i = 0; i< currencies.length; ++i) {
this.codeMap.put(currencies[i], i);
Expand All @@ -186,7 +187,7 @@ public OfCurrency() {

@Override
public final int getCode(Currency value) {
return value == null ? -1 : codeMap.get(value);
return value == null ? -1 : codeMap.getInt(value);
}

@Override
Expand All @@ -204,15 +205,16 @@ class OfZoneId extends BaseCoding<ZoneId> implements IntCoding<ZoneId> {
private static final long serialVersionUID = 1L;

private final ZoneId[] zoneIds;
private final TObjectIntMap<ZoneId> codeMap;
private final Object2IntMap<ZoneId> codeMap;

/**
* Constructor
*/
OfZoneId() {
super(ZoneId.class);
this.zoneIds = ZoneId.getAvailableZoneIds().stream().map(ZoneId::of).toArray(ZoneId[]::new);
this.codeMap = new TObjectIntHashMap<>(zoneIds.length, 0.5f, -1);
this.codeMap = new Object2IntOpenHashMap<>(zoneIds.length, 0.5f);
this.codeMap.defaultReturnValue(-1);
Arrays.sort(zoneIds, (z1, z2) -> z1.getId().compareTo(z2.getId()));
for (int i=0; i<zoneIds.length; ++i) {
this.codeMap.put(zoneIds[i], i);
Expand All @@ -221,7 +223,7 @@ class OfZoneId extends BaseCoding<ZoneId> implements IntCoding<ZoneId> {

@Override
public final int getCode(ZoneId value) {
return value == null ? -1 : codeMap.get(value);
return value == null ? -1 : codeMap.getInt(value);
}

@Override
Expand All @@ -239,15 +241,16 @@ class OfTimeZone extends BaseCoding<TimeZone> implements IntCoding<TimeZone> {
private static final long serialVersionUID = 1L;

private final TimeZone[] timeZones;
private final TObjectIntMap<TimeZone> codeMap;
private final Object2IntMap<TimeZone> codeMap;

/**
* Constructor
*/
OfTimeZone() {
super(TimeZone.class);
this.timeZones = Arrays.stream(TimeZone.getAvailableIDs()).map(TimeZone::getTimeZone).toArray(TimeZone[]::new);
this.codeMap = new TObjectIntHashMap<>(timeZones.length, 0.5f, -1);
this.codeMap = new Object2IntOpenHashMap<>(timeZones.length, 0.5f);
this.codeMap.defaultReturnValue(-1);
Arrays.sort(timeZones, (tz1, tz2) -> tz1.getID().compareTo(tz2.getID()));
for (int i = 0; i< timeZones.length; ++i) {
this.codeMap.put(timeZones[i], i);
Expand All @@ -256,7 +259,7 @@ class OfTimeZone extends BaseCoding<TimeZone> implements IntCoding<TimeZone> {

@Override
public final int getCode(TimeZone value) {
return value == null ? -1 : codeMap.get(value);
return value == null ? -1 : codeMap.getInt(value);
}

@Override
Expand Down
13 changes: 7 additions & 6 deletions src/main/java/com/zavtech/morpheus/array/dense/DenseArrayOfBooleans.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,17 @@
import java.util.Arrays;
import java.util.function.Predicate;

import gnu.trove.set.TShortSet;
import gnu.trove.set.hash.TShortHashSet;

import com.zavtech.morpheus.array.Array;
import com.zavtech.morpheus.array.ArrayBase;
import com.zavtech.morpheus.array.ArrayBuilder;
import com.zavtech.morpheus.array.ArrayCursor;
import com.zavtech.morpheus.array.ArrayException;
import com.zavtech.morpheus.array.Array;
import com.zavtech.morpheus.array.ArrayBase;
import com.zavtech.morpheus.array.ArrayStyle;
import com.zavtech.morpheus.array.ArrayValue;

import it.unimi.dsi.fastutil.shorts.ShortOpenHashSet;
import it.unimi.dsi.fastutil.shorts.ShortSet;

/**
* An Array implementation designed to hold a dense array of boolean values
*
Expand Down Expand Up @@ -281,7 +281,8 @@ public int binarySearch(int start, int end, Boolean value) {

@Override
public Array<Boolean> distinct(int limit) {
final TShortSet set = new TShortHashSet(limit);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final ShortSet set = new ShortOpenHashSet(capacity);
final ArrayBuilder<Boolean> builder = ArrayBuilder.of(2, Boolean.class);
for (int i=0; i<length(); ++i) {
final boolean value = getBoolean(i);
Expand Down
14 changes: 7 additions & 7 deletions src/main/java/com/zavtech/morpheus/array/dense/DenseArrayOfDoubles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,17 @@
import java.util.Arrays;
import java.util.function.Predicate;

import gnu.trove.set.TDoubleSet;
import gnu.trove.set.hash.TDoubleHashSet;

import com.zavtech.morpheus.array.Array;
import com.zavtech.morpheus.array.ArrayBase;
import com.zavtech.morpheus.array.ArrayBuilder;
import com.zavtech.morpheus.array.ArrayCursor;
import com.zavtech.morpheus.array.ArrayException;
import com.zavtech.morpheus.array.Array;
import com.zavtech.morpheus.array.ArrayBase;
import com.zavtech.morpheus.array.ArrayStyle;
import com.zavtech.morpheus.array.ArrayValue;

import it.unimi.dsi.fastutil.doubles.DoubleOpenHashSet;
import it.unimi.dsi.fastutil.doubles.DoubleSet;

/**
* An Array implementation designed to hold a dense array of double values
*
Expand Down Expand Up @@ -261,8 +261,8 @@ public final int binarySearch(int start, int end, Double value) {

@Override
public final Array<Double> distinct(int limit) {
final int capacity = limit < Integer.MAX_VALUE ? limit : 100;
final TDoubleSet set = new TDoubleHashSet(capacity);
final int capacity = limit < it.unimi.dsi.fastutil.Arrays.MAX_ARRAY_SIZE ? limit : 1000;
final DoubleSet set = new DoubleOpenHashSet(capacity);
final ArrayBuilder<Double> builder = ArrayBuilder.of(capacity, Double.class);
for (int i=0; i<length(); ++i) {
final double value = getDouble(i);
Expand Down
Loading