diff --git a/spi/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttribute.java b/spi/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttribute.java deleted file mode 100644 index d7f1a3b9..00000000 --- a/spi/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttribute.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright (c) 2023 Works Applications Co., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.worksap.nlp.lucene.sudachi.ja.attributes; - -import org.apache.lucene.util.Attribute; - -/** - * This attribute tells Sudachi-based TokenStreams not to produce anything into - * {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute} if it is - * not the current consumer.
- * This is performance optimisation and will not change correctness if resetting - * {@code CharTermAttribute} before writing into it. - */ -public interface MorphemeConsumerAttribute extends Attribute { - /** - * Check whether the object should consume the token stream. - * - * @param consumer - * object that will try to consume the token stream - * @return true if the object is current consumer - */ - default boolean shouldConsume(Object consumer) { - return consumer == getCurrentConsumer(); - } - - /** - * Get the current consumer - * - * @return instance that is current consumer - */ - Object getCurrentConsumer(); - - /** - * Set the current consumer for the token stream - * - * @param consumer - * new consumer instance - */ - void setCurrentConsumer(Object consumer); -} diff --git a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/MorphemeFieldFilter.kt b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/MorphemeFieldFilter.kt index 56bfa0cc..6ff4825b 100644 --- a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/MorphemeFieldFilter.kt +++ b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/MorphemeFieldFilter.kt @@ -17,9 +17,7 @@ package com.worksap.nlp.lucene.sudachi.ja import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeAttribute -import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeConsumerAttribute import com.worksap.nlp.sudachi.Morpheme -import org.apache.logging.log4j.LogManager import org.apache.lucene.analysis.TokenFilter import org.apache.lucene.analysis.TokenStream import org.apache.lucene.analysis.tokenattributes.CharTermAttribute @@ -39,8 +37,6 @@ abstract class MorphemeFieldFilter(input: TokenStream) : TokenFilter(input) { @JvmField protected val morphemeAtt = existingAttribute() @JvmField protected val keywordAtt = addAttribute() @JvmField protected val termAtt = addAttribute() - @JvmField - protected val consumer = addAttribute { it.currentConsumer = this } /** * Override this method to customize returned value. This method will not be called if @@ -64,16 +60,4 @@ abstract class MorphemeFieldFilter(input: TokenStream) : TokenFilter(input) { return true } - - override fun reset() { - super.reset() - if (!consumer.shouldConsume(this)) { - logger.warn( - "an instance of ${javaClass.name} is a no-op, it is not a filter which produces terms in one of your filter chains") - } - } - - companion object { - private val logger = LogManager.getLogger(MorphemeFieldFilter::class.java) - } } diff --git a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiSplitFilter.java b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiSplitFilter.java index ac668d95..361fd66e 100644 --- a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiSplitFilter.java +++ b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiSplitFilter.java @@ -86,7 +86,6 @@ public int offset() { private final PositionIncrementAttribute posIncAtt; private final PositionLengthAttribute posLengthAtt; private final MorphemeAttribute morphemeAtt; - private final MorphemeConsumerAttribute consumerAttribute; private ListIterator aUnitIterator; private final OovChars oovChars = new OovChars(); @@ -102,8 +101,6 @@ public SudachiSplitFilter(TokenStream input, Mode mode, Tokenizer.SplitMode spli posIncAtt = addAttribute(PositionIncrementAttribute.class); posLengthAtt = addAttribute(PositionLengthAttribute.class); morphemeAtt = addAttribute(MorphemeAttribute.class); - consumerAttribute = addAttribute(MorphemeConsumerAttribute.class); - consumerAttribute.setCurrentConsumer(this); } @Override diff --git a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiTokenizer.kt b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiTokenizer.kt index ef209caa..4bd8a44e 100644 --- a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiTokenizer.kt +++ b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/SudachiTokenizer.kt @@ -17,7 +17,6 @@ package com.worksap.nlp.lucene.sudachi.ja import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeAttribute -import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeConsumerAttribute import com.worksap.nlp.lucene.sudachi.ja.attributes.SudachiAttribute import com.worksap.nlp.lucene.sudachi.ja.attributes.SudachiAttributeFactory import org.apache.lucene.analysis.Tokenizer @@ -37,7 +36,6 @@ class SudachiTokenizer( private val offsetAtt = addAttribute() private val posIncAtt = addAttribute() private val posLenAtt = addAttribute() - private val consumer = addAttribute { it.currentConsumer = this } init { addAttribute { it.dictionary = tokenizer.dictionary } diff --git a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttributeImpl.kt b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttributeImpl.kt deleted file mode 100644 index 03ca4d99..00000000 --- a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/MorphemeConsumerAttributeImpl.kt +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2022-2023 Works Applications Co., Ltd. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.worksap.nlp.lucene.sudachi.ja.attributes - -import com.worksap.nlp.lucene.sudachi.ja.reflect -import org.apache.lucene.util.AttributeImpl -import org.apache.lucene.util.AttributeReflector - -/** - * Sudachi-based TokenStream chain uses to communicate which component produces - * [org.apache.lucene.analysis.tokenattributes.CharTermAttribute] - * - * This is not a token-based attribute, so impl's clear/copyTo do nothing - */ -class MorphemeConsumerAttributeImpl : AttributeImpl(), MorphemeConsumerAttribute { - private var instance: Any = Companion - // does nothing - override fun clear() {} - - override fun reflectWith(reflector: AttributeReflector) { - reflector.reflect("instance", instance.javaClass.name) - } - - override fun copyTo(target: AttributeImpl?) {} - - override fun getCurrentConsumer(): Any = instance - - override fun setCurrentConsumer(consumer: Any?) { - instance = consumer!! - } - - // need something to use as initial value of [instance] variable - private companion object -} diff --git a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/SudachiAttributeFactory.kt b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/SudachiAttributeFactory.kt index 0ae1c19e..f21a2263 100644 --- a/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/SudachiAttributeFactory.kt +++ b/src/main/java/com/worksap/nlp/lucene/sudachi/ja/attributes/SudachiAttributeFactory.kt @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Works Applications Co., Ltd. + * Copyright (c) 2023-2024 Works Applications Co., Ltd. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,7 +25,6 @@ class SudachiAttributeFactory(private val parent: AttributeFactory) : AttributeF override fun createAttributeInstance(attClass: Class?): AttributeImpl { return when (attClass) { MorphemeAttribute::class.java -> MorphemeAttributeImpl() - MorphemeConsumerAttribute::class.java -> MorphemeConsumerAttributeImpl() SudachiAttribute::class.java -> SudachiAttributeImpl() else -> parent.createAttributeInstance(attClass) }