Skip to content

Commit

Permalink
rm MorphemeConsumerAttribute
Browse files Browse the repository at this point in the history
  • Loading branch information
mh-northlander committed May 30, 2024
1 parent 83102de commit 3b2163c
Show file tree
Hide file tree
Showing 6 changed files with 1 addition and 125 deletions.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,7 @@
package com.worksap.nlp.lucene.sudachi.ja

import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeAttribute
import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeConsumerAttribute
import com.worksap.nlp.sudachi.Morpheme
import org.apache.logging.log4j.LogManager
import org.apache.lucene.analysis.TokenFilter
import org.apache.lucene.analysis.TokenStream
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute
Expand All @@ -39,8 +37,6 @@ abstract class MorphemeFieldFilter(input: TokenStream) : TokenFilter(input) {
@JvmField protected val morphemeAtt = existingAttribute<MorphemeAttribute>()
@JvmField protected val keywordAtt = addAttribute<KeywordAttribute>()
@JvmField protected val termAtt = addAttribute<CharTermAttribute>()
@JvmField
protected val consumer = addAttribute<MorphemeConsumerAttribute> { it.currentConsumer = this }

/**
* Override this method to customize returned value. This method will not be called if
Expand All @@ -64,16 +60,4 @@ abstract class MorphemeFieldFilter(input: TokenStream) : TokenFilter(input) {

return true
}

override fun reset() {
super.reset()
if (!consumer.shouldConsume(this)) {
logger.warn(
"an instance of ${javaClass.name} is a no-op, it is not a filter which produces terms in one of your filter chains")
}
}

companion object {
private val logger = LogManager.getLogger(MorphemeFieldFilter::class.java)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,6 @@ public int offset() {
private final PositionIncrementAttribute posIncAtt;
private final PositionLengthAttribute posLengthAtt;
private final MorphemeAttribute morphemeAtt;
private final MorphemeConsumerAttribute consumerAttribute;
private ListIterator<Morpheme> aUnitIterator;
private final OovChars oovChars = new OovChars();

Expand All @@ -102,8 +101,6 @@ public SudachiSplitFilter(TokenStream input, Mode mode, Tokenizer.SplitMode spli
posIncAtt = addAttribute(PositionIncrementAttribute.class);
posLengthAtt = addAttribute(PositionLengthAttribute.class);
morphemeAtt = addAttribute(MorphemeAttribute.class);
consumerAttribute = addAttribute(MorphemeConsumerAttribute.class);
consumerAttribute.setCurrentConsumer(this);
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package com.worksap.nlp.lucene.sudachi.ja

import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeAttribute
import com.worksap.nlp.lucene.sudachi.ja.attributes.MorphemeConsumerAttribute
import com.worksap.nlp.lucene.sudachi.ja.attributes.SudachiAttribute
import com.worksap.nlp.lucene.sudachi.ja.attributes.SudachiAttributeFactory
import org.apache.lucene.analysis.Tokenizer
Expand All @@ -37,7 +36,6 @@ class SudachiTokenizer(
private val offsetAtt = addAttribute<OffsetAttribute>()
private val posIncAtt = addAttribute<PositionIncrementAttribute>()
private val posLenAtt = addAttribute<PositionLengthAttribute>()
private val consumer = addAttribute<MorphemeConsumerAttribute> { it.currentConsumer = this }

init {
addAttribute<SudachiAttribute> { it.dictionary = tokenizer.dictionary }
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2023 Works Applications Co., Ltd.
* Copyright (c) 2023-2024 Works Applications Co., Ltd.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -25,7 +25,6 @@ class SudachiAttributeFactory(private val parent: AttributeFactory) : AttributeF
override fun createAttributeInstance(attClass: Class<out Attribute>?): AttributeImpl {
return when (attClass) {
MorphemeAttribute::class.java -> MorphemeAttributeImpl()
MorphemeConsumerAttribute::class.java -> MorphemeConsumerAttributeImpl()
SudachiAttribute::class.java -> SudachiAttributeImpl()
else -> parent.createAttributeInstance(attClass)
}
Expand Down

0 comments on commit 3b2163c

Please sign in to comment.