From 1a0c8d340daa2b418ea318c307ef6c4820b7622e Mon Sep 17 00:00:00 2001 From: edu Date: Tue, 11 Oct 2016 11:52:51 +0200 Subject: [PATCH] Upgraded to cassandra 3.8 --- CHANGELOG.md | 5 +- builder/pom.xml | 2 +- doc/documentation.rst | 60 ++++++++++--------- plugin/pom.xml | 10 +++- .../com/stratio/cassandra/lucene/Index.java | 1 + .../cassandra/lucene/IndexService.java | 19 +++--- .../cassandra/lucene/IndexServiceSkinny.java | 2 +- .../cassandra/lucene/IndexServiceWide.java | 2 +- .../stratio/cassandra/lucene/IndexWriter.java | 4 -- .../cassandra/lucene/IndexWriterSkinny.java | 6 +- .../cassandra/lucene/IndexWriterWide.java | 5 +- .../lucene/column/ColumnsMapperTest.scala | 3 +- pom.xml | 2 +- testsAT/pom.xml | 6 +- 14 files changed, 65 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f41679c16..82126e84c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog +## 3.8.0 (Upcoming) -## 3.7.3 (Upcoming) +* Upgrade to Apache Cassandra 3.8 + +## 3.7.3 (October 07, 2016) * Add support for indexing non-frozen UDTs diff --git a/builder/pom.xml b/builder/pom.xml index bc6ff29eb..21d516d09 100644 --- a/builder/pom.xml +++ b/builder/pom.xml @@ -26,7 +26,7 @@ com.stratio.cassandra cassandra-lucene-index-parent - 3.7.3-RC1-SNAPSHOT + 3.8.0-SNAPSHOT jar diff --git a/doc/documentation.rst b/doc/documentation.rst index c273eaf7f..d9d596054 100644 --- a/doc/documentation.rst +++ b/doc/documentation.rst @@ -244,35 +244,37 @@ If you have huge amount of data in your cluster this could be an expensive task. compatibility matrix that states between which versions it is not needed to delete the index: -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| From\\ To | 3.0.3.0 | 3.0.3.1 | 3.0.4.0 | 3.0.4.1 | 3.0.5.0 | 3.5.0 | 3.5.1 | 3.5.2 | 3.6.0 | 3.7.0 | 3.7.1 | 3.7.2 | 3.7.3 | -+===========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+ -| 2.x | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.0.3.0 | -- | YES | YES | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.0.3.1 | -- | -- | YES | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.0.4.0 | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.0.4.1 | -- | -- | -- | -- | YES | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.0.5.0 | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.5.0 | -- | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.5.1 | -- | -- | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.5.2 | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | YES | YES | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.6.0 | -- | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | YES | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.7.0 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.7.1 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | (1) | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ -| 3.7.2 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | NO | -+-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| From\\ To | 3.0.3.0 | 3.0.3.1 | 3.0.4.0 | 3.0.4.1 | 3.0.5.0 | 3.5.0 | 3.5.1 | 3.5.2 | 3.6.0 | 3.7.0 | 3.7.1 | 3.7.2 | 3.7.3 | 3.8.0 | ++===========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+=========+ +| 2.x | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.0.3.0 | -- | YES | YES | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.0.3.1 | -- | -- | YES | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.0.4.0 | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.0.4.1 | -- | -- | -- | -- | YES | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.0.5.0 | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.5.0 | -- | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.5.1 | -- | -- | -- | -- | -- | -- | -- | NO | NO | NO | NO | NO | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.5.2 | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | YES | YES | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.6.0 | -- | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | YES | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.7.0 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | YES | YES | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.7.1 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | (1) | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.7.2 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | NO | NO | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ +| 3.7.3 | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | -- | YES | ++-----------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+---------+ **(1):** Compatible only if you are not using geospatial mappers. diff --git a/plugin/pom.xml b/plugin/pom.xml index 0a3a73e87..5a00f7f1b 100644 --- a/plugin/pom.xml +++ b/plugin/pom.xml @@ -26,7 +26,7 @@ com.stratio.cassandra cassandra-lucene-index-parent - 3.7.3-RC1-SNAPSHOT + 3.8.0-SNAPSHOT jar @@ -38,7 +38,7 @@ 2.10.4 2.10 2.2.5 - 3.7 + 3.8 5.5.1 1.14.0 1.10.19 @@ -99,6 +99,12 @@ ${mockito.version} test + + junit + junit + 4.12 + test + diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/Index.java b/plugin/src/main/java/com/stratio/cassandra/lucene/Index.java index d4e1c1da8..65ff62773 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/Index.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/Index.java @@ -199,6 +199,7 @@ public Callable getBlockingFlushTask() { return () -> { logger.info("Flushing Lucene index {}", name); service.commit(); + logger.info("Flushed Lucene index {}", name); return null; }; } diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexService.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexService.java index 3f185e2b5..6818af8a6 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexService.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexService.java @@ -16,9 +16,7 @@ package com.stratio.cassandra.lucene; import com.stratio.cassandra.lucene.column.Columns; -import com.stratio.cassandra.lucene.index.DocumentIterator; -import com.stratio.cassandra.lucene.index.FSIndex; -import com.stratio.cassandra.lucene.index.RAMIndex; +import com.stratio.cassandra.lucene.index.*; import com.stratio.cassandra.lucene.key.PartitionMapper; import com.stratio.cassandra.lucene.key.TokenMapper; import com.stratio.cassandra.lucene.schema.Schema; @@ -592,17 +590,17 @@ private Sort sort(Search search) { * @param key the partition key * @param clusterings the clustering keys * @param nowInSec max allowed time in seconds - * @param opGroup operation group spanning the calling operation * @return a {@link Row} iterator */ UnfilteredRowIterator read(DecoratedKey key, NavigableSet clusterings, - int nowInSec, - OpOrder.Group opGroup) { + int nowInSec) { ClusteringIndexNamesFilter filter = new ClusteringIndexNamesFilter(clusterings, false); ColumnFilter columnFilter = ColumnFilter.all(metadata); - return SinglePartitionReadCommand.create(metadata, nowInSec, key, columnFilter, filter) - .queryMemtableAndDisk(table, opGroup); + SinglePartitionReadCommand readCommand= SinglePartitionReadCommand.create(metadata, nowInSec, key, columnFilter, filter); + try(ReadExecutionController controller=readCommand.executionController()) { + return readCommand.queryMemtableAndDisk(table,controller); + } } /** @@ -610,11 +608,10 @@ UnfilteredRowIterator read(DecoratedKey key, * * @param key the partition key * @param nowInSec max allowed time in seconds - * @param opGroup operation group spanning the calling operation * @return a {@link Row} iterator */ - UnfilteredRowIterator read(DecoratedKey key, int nowInSec, OpOrder.Group opGroup) { - return read(key, clusterings(Clustering.EMPTY), nowInSec, opGroup); + UnfilteredRowIterator read(DecoratedKey key, int nowInSec) { + return read(key, clusterings(Clustering.EMPTY), nowInSec); } /** diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceSkinny.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceSkinny.java index f4b91bdfe..838469ae2 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceSkinny.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceSkinny.java @@ -71,7 +71,7 @@ public IndexWriterSkinny indexWriter(DecoratedKey key, int nowInSec, OpOrder.Group opGroup, IndexTransaction.Type transactionType) { - return new IndexWriterSkinny(this, key, nowInSec, opGroup, transactionType); + return new IndexWriterSkinny(this, key, nowInSec, transactionType); } /** {@inheritDoc} */ diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceWide.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceWide.java index d79a8d24c..fd74a4a9a 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceWide.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexServiceWide.java @@ -94,7 +94,7 @@ public IndexWriterWide indexWriter(DecoratedKey key, int nowInSec, OpOrder.Group opGroup, IndexTransaction.Type transactionType) { - return new IndexWriterWide(this, key, nowInSec, opGroup, transactionType); + return new IndexWriterWide(this, key, nowInSec, transactionType); } /** {@inheritDoc} */ diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriter.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriter.java index c45d00457..cddbca2ae 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriter.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriter.java @@ -36,7 +36,6 @@ abstract class IndexWriter implements Index.Indexer { protected final IndexService service; protected final DecoratedKey key; protected final int nowInSec; - protected final OpOrder.Group opGroup; protected final IndexTransaction.Type transactionType; /** @@ -45,18 +44,15 @@ abstract class IndexWriter implements Index.Indexer { * @param service the service to perform the indexing operation * @param key key of the partition being modified * @param nowInSec current time of the update operation - * @param opGroup operation group spanning the update operation * @param transactionType what kind of update is being performed on the base data */ IndexWriter(IndexService service, DecoratedKey key, int nowInSec, - OpOrder.Group opGroup, IndexTransaction.Type transactionType) { this.service = service; this.key = key; this.nowInSec = nowInSec; - this.opGroup = opGroup; this.transactionType = transactionType; } diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterSkinny.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterSkinny.java index 334111130..1c2225fd0 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterSkinny.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterSkinny.java @@ -39,15 +39,13 @@ class IndexWriterSkinny extends IndexWriter { * @param service the service to perform the indexing operation * @param key key of the partition being modified * @param nowInSec current time of the update operation - * @param opGroup operation group spanning the update operation * @param transactionType what kind of update is being performed on the base data */ IndexWriterSkinny(IndexServiceSkinny service, DecoratedKey key, int nowInSec, - OpOrder.Group opGroup, IndexTransaction.Type transactionType) { - super(service, key, nowInSec, opGroup, transactionType); + super(service, key, nowInSec, transactionType); optionalRow = Optional.empty(); } @@ -71,7 +69,7 @@ public void finish() { optionalRow.ifPresent(row -> { if (transactionType == IndexTransaction.Type.COMPACTION || service.needsReadBeforeWrite(key, row)) { Tracer.trace("Lucene index reading before write"); - UnfilteredRowIterator iterator = service.read(key, nowInSec, opGroup); + UnfilteredRowIterator iterator = service.read(key, nowInSec); if (iterator.hasNext()) { row = (Row) iterator.next(); } diff --git a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterWide.java b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterWide.java index 9205b498f..3aebdb55f 100644 --- a/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterWide.java +++ b/plugin/src/main/java/com/stratio/cassandra/lucene/IndexWriterWide.java @@ -49,9 +49,8 @@ class IndexWriterWide extends IndexWriter { IndexWriterWide(IndexServiceWide service, DecoratedKey key, int nowInSec, - OpOrder.Group opGroup, IndexTransaction.Type transactionType) { - super(service, key, nowInSec, opGroup, transactionType); + super(service, key, nowInSec, transactionType); rowsToRead = service.clusterings(); rows = new LinkedHashMap<>(); } @@ -86,7 +85,7 @@ public void finish() { if (transactionType != IndexTransaction.Type.CLEANUP) { // Read required rows from storage engine - service.read(key, rowsToRead, nowInSec, opGroup).forEachRemaining(unfiltered -> { + service.read(key, rowsToRead, nowInSec).forEachRemaining(unfiltered -> { Row row = (Row) unfiltered; rows.put(row.clustering(), Optional.of(row)); }); diff --git a/plugin/src/test/scala/com/stratio/cassandra/lucene/column/ColumnsMapperTest.scala b/plugin/src/test/scala/com/stratio/cassandra/lucene/column/ColumnsMapperTest.scala index d30810f42..574355b3a 100644 --- a/plugin/src/test/scala/com/stratio/cassandra/lucene/column/ColumnsMapperTest.scala +++ b/plugin/src/test/scala/com/stratio/cassandra/lucene/column/ColumnsMapperTest.scala @@ -23,6 +23,7 @@ import com.google.common.collect.Lists import com.stratio.cassandra.lucene.BaseScalaTest import com.stratio.cassandra.lucene.column.ColumnsMapper._ import org.apache.cassandra.config.ColumnDefinition +import org.apache.cassandra.cql3.FieldIdentifier import org.apache.cassandra.db.marshal._ import org.apache.cassandra.db.rows.{BufferCell, Cell} import org.apache.cassandra.utils.UUIDGen @@ -252,7 +253,7 @@ class ColumnsMapperTest extends BaseScalaTest { new UserType( "ks", UTF8Type.instance.decompose("cell"), - Lists.newArrayList(names.map(x => UTF8Type.instance.decompose(x)).asJava), + Lists.newArrayList(names.map(x => new FieldIdentifier(UTF8Type.instance.decompose(x))).asJava), Lists.newArrayList(types.asJava),false) } } diff --git a/pom.xml b/pom.xml index 3c0956ce3..94fb359eb 100644 --- a/pom.xml +++ b/pom.xml @@ -32,7 +32,7 @@ pom com.stratio.cassandra cassandra-lucene-index-parent - 3.7.3-RC1-SNAPSHOT + 3.8.0-SNAPSHOT Cassandra Lucene index Cassandra Lucene index diff --git a/testsAT/pom.xml b/testsAT/pom.xml index e64709a78..a212e805b 100644 --- a/testsAT/pom.xml +++ b/testsAT/pom.xml @@ -25,7 +25,7 @@ com.stratio.cassandra cassandra-lucene-index-tests - 3.7.3-RC1-SNAPSHOT + 3.8.0-SNAPSHOT jar Cassandra Lucene Index acceptance tests @@ -34,7 +34,7 @@ 1.8 1.8 - 3.0.2 + 3.1.0 UTF-8 **/AllSuite.java @@ -67,7 +67,7 @@ junit junit - 4.8.1 + 4.12 test