From 053db3b67f50deac123cdcf77ad7df8cb9299815 Mon Sep 17 00:00:00 2001 From: Toshiyuki Hanaoka Date: Thu, 26 Oct 2023 09:07:31 +0000 Subject: [PATCH] Add a feature to enable findability oriented candidate order. PiperOrigin-RevId: 576789852 --- src/converter/segments.h | 8 + src/converter/segments_matchers.h | 1 + src/protocol/commands.proto | 7 +- src/rewriter/BUILD.bazel | 32 +++ src/rewriter/date_rewriter.cc | 1 + src/rewriter/emoji_rewriter.cc | 1 + src/rewriter/emoticon_rewriter.cc | 1 + src/rewriter/order_rewriter.cc | 222 +++++++++++++++++++ src/rewriter/order_rewriter.h | 51 +++++ src/rewriter/order_rewriter_test.cc | 147 ++++++++++++ src/rewriter/rewriter.cc | 2 + src/rewriter/rewriter.gyp | 1 + src/rewriter/rewriter_test.gyp | 1 + src/rewriter/symbol_rewriter.cc | 1 + src/rewriter/t13n_promotion_rewriter.cc | 5 + src/session/session_converter.cc | 9 +- src/session/session_converter_test.cc | 33 +++ src/session/session_handler_scenario_test.cc | 6 + 18 files changed, 525 insertions(+), 4 deletions(-) create mode 100644 src/rewriter/order_rewriter.cc create mode 100644 src/rewriter/order_rewriter.h create mode 100644 src/rewriter/order_rewriter_test.cc diff --git a/src/converter/segments.h b/src/converter/segments.h index 9026e7895..45e287de7 100644 --- a/src/converter/segments.h +++ b/src/converter/segments.h @@ -156,6 +156,12 @@ class Segment final { USER_HISTORY_PREDICTOR = 1 << 6, }; + enum Category { + DEFAULT_CATEGORY, // Realtime conversion, history prediction, etc + SYMBOL, // Symbol, emoji + OTHER, // Misc candidate + }; + // LINT.IfChange std::string key; // reading std::string value; // surface form @@ -202,6 +208,8 @@ class Segment final { // Candidate's source info which will be used for usage stats. uint32_t source_info = SOURCE_INFO_NONE; + Category category = DEFAULT_CATEGORY; + // Candidate style. This is not a bit-field. // The style is defined in enum |Style|. NumberUtil::NumberString::Style style = diff --git a/src/converter/segments_matchers.h b/src/converter/segments_matchers.h index 8fa6aaec7..f43019123 100644 --- a/src/converter/segments_matchers.h +++ b/src/converter/segments_matchers.h @@ -73,6 +73,7 @@ MATCHER_P(EqualsCandidate, candidate, "") { COMPARE_FIELD(rid); COMPARE_FIELD(attributes); COMPARE_FIELD(source_info); + COMPARE_FIELD(category); COMPARE_FIELD(style); COMPARE_FIELD(command); COMPARE_FIELD(inner_segment_boundary); diff --git a/src/protocol/commands.proto b/src/protocol/commands.proto index 77b6629fa..840fd6e91 100644 --- a/src/protocol/commands.proto +++ b/src/protocol/commands.proto @@ -556,7 +556,7 @@ message Capability { [default = NO_TEXT_DELETION_CAPABILITY]; } -// Next ID: 42 +// Next ID: 43 // Bundles together some Android experiment flags so that they can be easily // retrieved throughout the native code. These flags are generally specific to // the decoder, and are made available when the decoder is initialized. @@ -619,12 +619,13 @@ message DecoderExperimentParams { reserved 20; // Deprecated cancel_content_word_suffix_penalty reserved 23; // Deprecated typing_correction_score_offset reserved 24; // Deprecated typing_correction_move_literal_candidate_to_top + reserved 30; // Deprecated enable_number_style_learning optional bool disable_zero_query_suffix_prediction = 36 [default = false]; - reserved 30; // Deprecated enable_number_style_learning - optional bool enable_realtime_conversion_v2 = 37 [default = false]; + + optional bool enable_findability_oriented_order = 42 [default = false]; } // Clients' request to the server. diff --git a/src/rewriter/BUILD.bazel b/src/rewriter/BUILD.bazel index 247239e04..f21ab9bfa 100644 --- a/src/rewriter/BUILD.bazel +++ b/src/rewriter/BUILD.bazel @@ -1414,6 +1414,7 @@ mozc_cc_library( ":language_aware_rewriter", ":merger_rewriter", ":number_rewriter", + ":order_rewriter", ":remove_redundant_candidate_rewriter", ":rewriter_interface", ":single_kanji_rewriter", @@ -1606,3 +1607,34 @@ mozc_cc_test( "//testing:gunit_main", ], ) + +mozc_cc_library( + name = "order_rewriter", + srcs = ["order_rewriter.cc"], + hdrs = ["order_rewriter.h"], + deps = [ + ":rewriter_interface", + "//base:util", + "//converter:segments", + "//request:conversion_request", + "@com_google_absl//absl/algorithm:container", + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@com_google_absl//absl/log", + ], +) + +mozc_cc_test( + name = "order_rewriter_test", + srcs = ["order_rewriter_test.cc"], + deps = [ + ":order_rewriter", + ":rewriter_interface", + "//converter:segments", + "//converter:segments_matchers", + "//request:conversion_request", + "//session:request_test_util", + "//testing:gunit_main", + "//testing:mozctest", + ], +) diff --git a/src/rewriter/date_rewriter.cc b/src/rewriter/date_rewriter.cc index 751874477..ef1f1449d 100644 --- a/src/rewriter/date_rewriter.cc +++ b/src/rewriter/date_rewriter.cc @@ -512,6 +512,7 @@ std::unique_ptr CreateCandidate( candidate->content_key = base_candidate.content_key; candidate->attributes |= (Segment::Candidate::NO_LEARNING | Segment::Candidate::NO_VARIANTS_EXPANSION); + candidate->category = Segment::Candidate::OTHER; candidate->description = std::move(description); return candidate; } diff --git a/src/rewriter/emoji_rewriter.cc b/src/rewriter/emoji_rewriter.cc index 20c17463f..f72de6f18 100644 --- a/src/rewriter/emoji_rewriter.cc +++ b/src/rewriter/emoji_rewriter.cc @@ -84,6 +84,7 @@ std::unique_ptr CreateCandidate( } candidate->attributes |= Segment::Candidate::NO_VARIANTS_EXPANSION; candidate->attributes |= Segment::Candidate::CONTEXT_SENSITIVE; + candidate->category = Segment::Candidate::SYMBOL; return candidate; } diff --git a/src/rewriter/emoticon_rewriter.cc b/src/rewriter/emoticon_rewriter.cc index 42cc69ce5..284cdebcf 100644 --- a/src/rewriter/emoticon_rewriter.cc +++ b/src/rewriter/emoticon_rewriter.cc @@ -140,6 +140,7 @@ void InsertCandidates(SerializedDictionary::const_iterator begin, sorted_value[i].description().size()); c->description = description; } + c->category = Segment::Candidate::SYMBOL; } } diff --git a/src/rewriter/order_rewriter.cc b/src/rewriter/order_rewriter.cc new file mode 100644 index 000000000..3bf5050cb --- /dev/null +++ b/src/rewriter/order_rewriter.cc @@ -0,0 +1,222 @@ +// Copyright 2010-2021, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "rewriter/order_rewriter.h" + +#include +#include +#include +#include +#include +#include + +#include "base/util.h" +#include "converter/segments.h" +#include "request/conversion_request.h" +#include "rewriter/rewriter_interface.h" +#include "absl/algorithm/container.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +namespace mozc { + +namespace { + +class CandidateGroup { + public: + CandidateGroup() = default; + explicit CandidateGroup(Segment::Candidate::Category category) + : category_(category) {} + ~CandidateGroup() = default; + + const std::deque &candidates() const { + return candidates_; + } + + void AppendToSegment(Segment &segment) const { + for (const Segment::Candidate &c : candidates_) { + Segment::Candidate *candidate = segment.add_candidate(); + *candidate = c; + } + } + + std::deque *mutable_candidates() { return &candidates_; } + size_t size() const { return candidates_.size(); } + + void AddCandidate(const Segment::Candidate &candidate) { + if (const auto [_, inserted] = + added_.emplace(candidate.key, candidate.value); + inserted) { + candidates_.push_back(candidate); + if (category_.has_value()) { + candidates_.back().category = *category_; + } + } + } + + void AddHiragnaCandidates() { + // (Key, Base candidate) + absl::flat_hash_map keys; + for (const Segment::Candidate &c : candidates_) { + keys.insert({c.key, &c}); + } + for (const auto &itr : keys) { + Segment::Candidate c = *itr.second; + c.value = c.key; + c.content_key = c.key; + c.content_value = c.key; + c.description.clear(); + c.inner_segment_boundary.clear(); + + candidates_.push_front(std::move(c)); + } + } + + void SortCandidates() { + // key length -> value length + const auto cmp = [](const Segment::Candidate &lhs, + const Segment::Candidate &rhs) { + if (lhs.key.size() != rhs.key.size()) { + if (lhs.content_key.size() != rhs.content_key.size()) { + return lhs.content_key.size() > rhs.content_key.size(); + } + return lhs.key.size() > rhs.key.size(); + } + if (lhs.key != rhs.key) { + return lhs.key < rhs.key; + } + const size_t lhs_len = Util::CharsLen(lhs.value); + const size_t rhs_len = Util::CharsLen(rhs.value); + return lhs_len > rhs_len; + }; + absl::c_stable_sort(candidates_, cmp); + } + + private: + const std::optional category_ = std::nullopt; + std::deque candidates_; + absl::flat_hash_set> added_; +}; + +} // namespace + +int OrderRewriter::capability(const ConversionRequest &request) const { + if (request.request().mixed_conversion()) { // For mobile + return RewriterInterface::PREDICTION | RewriterInterface::SUGGESTION; + } else { + return RewriterInterface::NOT_AVAILABLE; + } +} + +bool OrderRewriter::Rewrite(const ConversionRequest &request, + Segments *segments) const { + if (!request.request() + .decoder_experiment_params() + .enable_findability_oriented_order()) { + return false; + } + if (segments->conversion_segments_size() != 1) { + return false; + } + + Segment *segment = segments->mutable_conversion_segment(0); + + // Candidates in the same category will be deduped. + CandidateGroup top, normal, partial, t13n; + CandidateGroup single_kanji(Segment::Candidate::OTHER), + single_kanji_partial(Segment::Candidate::OTHER), + symbol(Segment::Candidate::OTHER), other(Segment::Candidate::OTHER); + + constexpr int kTopCandidatesSize = 5; + for (size_t i = 0; i < segment->candidates_size(); ++i) { + const Segment::Candidate &candidate = segment->candidate(i); + if (top.size() < kTopCandidatesSize && + candidate.category != Segment::Candidate::OTHER) { + top.AddCandidate(candidate); + continue; + } + if (candidate.category == Segment::Candidate::SYMBOL) { + symbol.AddCandidate(candidate); + } else if (candidate.category == Segment::Candidate::OTHER) { + other.AddCandidate(candidate); + } else if (candidate.category == Segment::Candidate::DEFAULT_CATEGORY) { + // TODO(toshiyuki): Use better way to check single kanji entries. + // - There are single characters with multiple code points + // (e.g. SVS, IVS). Grapheme treats those multiple code points as a single + // character, but CharsLen() treats them as multiple characters. + // - Or, we may be able to set candidate category when we generate single + // kanji candidates. + const bool is_single_kanji = + Util::CharsLen(candidate.value) == 1 && + Util::IsScriptType(candidate.value, Util::KANJI); + const bool is_partial = + candidate.attributes & Segment::Candidate::PARTIALLY_KEY_CONSUMED; + if (is_partial && is_single_kanji) { + single_kanji_partial.AddCandidate(candidate); + } else if (is_partial) { + partial.AddCandidate(candidate); + } else if (is_single_kanji) { + single_kanji.AddCandidate(candidate); + } else { + normal.AddCandidate(candidate); + } + } + } + for (size_t i = 0; i < segment->meta_candidates_size(); ++i) { + const Segment::Candidate &c = segment->meta_candidate(i); + t13n.AddCandidate(c); + } + + single_kanji.AddHiragnaCandidates(); + single_kanji_partial.AddHiragnaCandidates(); + + // The following candidates are originally sorted in LM based order. + // Reorder these candidates based on the length of key and value so that + // users can find the expected candidate easily. + normal.SortCandidates(); + partial.SortCandidates(); + single_kanji.SortCandidates(); + single_kanji_partial.SortCandidates(); + + segment->clear_candidates(); + segment->clear_meta_candidates(); + + top.AppendToSegment(*segment); + normal.AppendToSegment(*segment); + t13n.AppendToSegment(*segment); + other.AppendToSegment(*segment); + single_kanji.AppendToSegment(*segment); + symbol.AppendToSegment(*segment); + partial.AppendToSegment(*segment); + single_kanji_partial.AppendToSegment(*segment); + + return true; +} + +} // namespace mozc diff --git a/src/rewriter/order_rewriter.h b/src/rewriter/order_rewriter.h new file mode 100644 index 000000000..75a38356f --- /dev/null +++ b/src/rewriter/order_rewriter.h @@ -0,0 +1,51 @@ +// Copyright 2010-2021, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#ifndef MOZC_REWRITER_ORDER_REWRITER_H_ +#define MOZC_REWRITER_ORDER_REWRITER_H_ + +#include "converter/segments.h" +#include "request/conversion_request.h" +#include "rewriter/rewriter_interface.h" + +namespace mozc { + +class OrderRewriter : public RewriterInterface { + public: + OrderRewriter() = default; + ~OrderRewriter() override = default; + + int capability(const ConversionRequest &request) const override; + bool Rewrite(const ConversionRequest &request, + Segments *segments) const override; +}; + +} // namespace mozc + +#endif // MOZC_REWRITER_ORDER_REWRITER_H_ diff --git a/src/rewriter/order_rewriter_test.cc b/src/rewriter/order_rewriter_test.cc new file mode 100644 index 000000000..b1591c4fe --- /dev/null +++ b/src/rewriter/order_rewriter_test.cc @@ -0,0 +1,147 @@ +// Copyright 2010-2021, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include "rewriter/order_rewriter.h" + +#include +#include + +#include "converter/segments.h" +#include "converter/segments_matchers.h" +#include "request/conversion_request.h" +#include "rewriter/rewriter_interface.h" +#include "session/request_test_util.h" +#include "testing/gmock.h" +#include "testing/gunit.h" +#include "testing/mozctest.h" + +namespace mozc { +namespace { + +using ::testing::Field; +using ::testing::Pointee; + +class OrderRewriterTest : public testing::TestWithTempUserProfile { + protected: + void SetUp() override { + rewriter_ = std::make_unique(); + + convreq_ = ConversionRequest(); + request_ = commands::Request(); + + convreq_.set_request(&request_); + } + + std::unique_ptr rewriter_; + ConversionRequest convreq_; + commands::Request request_; +}; + +TEST_F(OrderRewriterTest, Capability) { + // Desktop + EXPECT_EQ(rewriter_->capability(convreq_), RewriterInterface::NOT_AVAILABLE); + + // Mobile + commands::RequestForUnitTest::FillMobileRequest(&request_); + EXPECT_EQ(rewriter_->capability(convreq_), + RewriterInterface::PREDICTION | RewriterInterface::SUGGESTION); +} + +Segments BuildTestSegments() { + Segments segments; + segments.add_segment(); + auto add_candidate = [&](const std::string key, const std::string value, + Segment::Candidate::Category category) { + Segment *segment = segments.mutable_conversion_segment(0); + Segment::Candidate *c = segment->add_candidate(); + c->key = key; + c->content_key = key; + c->value = value; + c->content_value = value; + c->category = category; + if (c->key.size() < segment->key().size()) { + c->attributes = Segment::Candidate::PARTIALLY_KEY_CONSUMED; + c->consumed_key_size = c->key.size(); + } + }; + + segments.mutable_conversion_segment(0)->set_key("きょうの"); + add_candidate("きょうの", "今日の", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "きょうの", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "other", Segment::Candidate::OTHER); + add_candidate("きょうの", "教の", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "強の", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "凶の", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "キョウの", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "キョウノ", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょう", "今日", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょう", "きょう", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょう", "京", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょう", "供", Segment::Candidate::DEFAULT_CATEGORY); + add_candidate("きょうの", "😀", Segment::Candidate::SYMBOL); + add_candidate("きょうの", "響野", Segment::Candidate::DEFAULT_CATEGORY); + + return segments; +} + +TEST_F(OrderRewriterTest, NotAvailable) { + Segments segments = BuildTestSegments(); + EXPECT_FALSE(rewriter_->Rewrite(convreq_, &segments)); +} + +TEST_F(OrderRewriterTest, Rewrite) { + Segments segments = BuildTestSegments(); + request_.mutable_decoder_experiment_params() + ->set_enable_findability_oriented_order(true); + EXPECT_TRUE(rewriter_->Rewrite(convreq_, &segments)); + + constexpr auto ValueIs = [](const auto &value) { + return Pointee(Field(&Segment::Candidate::value, value)); + }; + EXPECT_THAT(segments.conversion_segment(0), CandidatesAreArray({ + ValueIs("今日の"), + ValueIs("きょうの"), + ValueIs("教の"), + ValueIs("強の"), + ValueIs("凶の"), + ValueIs("キョウの"), + ValueIs("キョウノ"), + ValueIs("響野"), + ValueIs("other"), + ValueIs("😀"), + ValueIs("きょう"), + ValueIs("今日"), + ValueIs("きょう"), + ValueIs("京"), + ValueIs("供"), + })); +} + +} // namespace +} // namespace mozc diff --git a/src/rewriter/rewriter.cc b/src/rewriter/rewriter.cc index f75141cc5..724d56d45 100644 --- a/src/rewriter/rewriter.cc +++ b/src/rewriter/rewriter.cc @@ -53,6 +53,7 @@ #include "rewriter/language_aware_rewriter.h" #include "rewriter/merger_rewriter.h" #include "rewriter/number_rewriter.h" +#include "rewriter/order_rewriter.h" #include "rewriter/remove_redundant_candidate_rewriter.h" #include "rewriter/rewriter_interface.h" #include "rewriter/single_kanji_rewriter.h" @@ -141,6 +142,7 @@ RewriterImpl::RewriterImpl(const ConverterInterface *parent_converter, AddRewriter(std::make_unique()); AddRewriter(std::make_unique(*data_manager)); AddRewriter(std::make_unique()); + AddRewriter(std::make_unique()); AddRewriter(std::make_unique(data_manager)); } diff --git a/src/rewriter/rewriter.gyp b/src/rewriter/rewriter.gyp index 6e4880766..8c0f00bf5 100644 --- a/src/rewriter/rewriter.gyp +++ b/src/rewriter/rewriter.gyp @@ -62,6 +62,7 @@ 'language_aware_rewriter.cc', 'number_compound_util.cc', 'number_rewriter.cc', + 'order_rewriter.cc', 'remove_redundant_candidate_rewriter.cc', 'rewriter.cc', 'rewriter_util.cc', diff --git a/src/rewriter/rewriter_test.gyp b/src/rewriter/rewriter_test.gyp index b9fc4d2b2..249a300fa 100644 --- a/src/rewriter/rewriter_test.gyp +++ b/src/rewriter/rewriter_test.gyp @@ -56,6 +56,7 @@ 'merger_rewriter_test.cc', 'number_compound_util_test.cc', 'number_rewriter_test.cc', + 'order_rewriter_test.cc', 'remove_redundant_candidate_rewriter_test.cc', 'rewriter_test.cc', 'small_letter_rewriter_test.cc', diff --git a/src/rewriter/symbol_rewriter.cc b/src/rewriter/symbol_rewriter.cc index 559113642..a722a2f78 100644 --- a/src/rewriter/symbol_rewriter.cc +++ b/src/rewriter/symbol_rewriter.cc @@ -223,6 +223,7 @@ void SymbolRewriter::InsertCandidates( candidate->value == "w" || candidate->value == "www") { candidate->attributes |= Segment::Candidate::NO_VARIANTS_EXPANSION; } + candidate->category = Segment::Candidate::SYMBOL; candidate->description = GetDescription( candidate->value, iter.description(), iter.additional_description()); diff --git a/src/rewriter/t13n_promotion_rewriter.cc b/src/rewriter/t13n_promotion_rewriter.cc index f3bc795e4..08d6bf17d 100644 --- a/src/rewriter/t13n_promotion_rewriter.cc +++ b/src/rewriter/t13n_promotion_rewriter.cc @@ -142,6 +142,11 @@ bool MaybePromoteT13n(const ConversionRequest &request, Segment *segment) { if (IsLatinInputMode(request) || Util::IsAscii(segment->key())) { return MaybeInsertLatinT13n(segment); } + if (request.request() + .decoder_experiment_params() + .enable_findability_oriented_order()) { + return false; + } return MaybePromoteKatakana(segment); } diff --git a/src/session/session_converter.cc b/src/session/session_converter.cc index 0330f076e..10cc9499a 100644 --- a/src/session/session_converter.cc +++ b/src/session/session_converter.cc @@ -1426,7 +1426,14 @@ void SessionConverter::AppendCandidateList() { const Segment &segment = segments_->conversion_segment(segment_index_); for (size_t i = candidate_list_->next_available_id(); i < segment.candidates_size(); ++i) { - candidate_list_->AddCandidate(i, segment.candidate(i).value); + if (request_->decoder_experiment_params() + .enable_findability_oriented_order()) { + const Segment::Candidate &c = segment.candidate(i); + candidate_list_->AddCandidate(i, + absl::StrCat(c.key, c.value, c.category)); + } else { + candidate_list_->AddCandidate(i, segment.candidate(i).value); + } // if candidate has spelling correction attribute, // always display the candidate to let user know the // miss spelled candidate. diff --git a/src/session/session_converter_test.cc b/src/session/session_converter_test.cc index c55526376..8e05bfde5 100644 --- a/src/session/session_converter_test.cc +++ b/src/session/session_converter_test.cc @@ -2379,6 +2379,39 @@ TEST_F(SessionConverterTest, AppendCandidateList) { } } +TEST_F(SessionConverterTest, AppendCandidateListWithCategory) { + MockConverter mock_converter; + request_->mutable_decoder_experiment_params() + ->set_enable_findability_oriented_order(true); + SessionConverter converter(&mock_converter, request_.get(), config_.get()); + SetState(SessionConverterInterface::PREDICTION, &converter); + Segments segments; + + { + SetAiueo(&segments); + FillT13Ns(&segments, composer_.get()); + + SetSegments(segments, &converter); + + Segment *segment = segments.mutable_conversion_segment(0); + Segment::Candidate *candidate = segment->add_candidate(); + candidate->key = "あいうえお"; + candidate->value = "あいうえお"; + candidate->category = Segment::Candidate::OTHER; + + candidate = segment->add_candidate(); + candidate->key = "あいうえお"; + candidate->value = "あいうえお"; + + AppendCandidateList(ConversionRequest::SUGGESTION, &converter); + const CandidateList &candidate_list = GetCandidateList(converter); + + // default あいうえお, default アイウエオ, other あいうえお + EXPECT_EQ(candidate_list.size(), 3); + EXPECT_FALSE(candidate_list.focused()); + } +} + TEST_F(SessionConverterTest, AppendCandidateListForRequestTypes) { MockConverter mock_converter; SessionConverter converter(&mock_converter, request_.get(), config_.get()); diff --git a/src/session/session_handler_scenario_test.cc b/src/session/session_handler_scenario_test.cc index 82fddabe4..77a267d79 100644 --- a/src/session/session_handler_scenario_test.cc +++ b/src/session/session_handler_scenario_test.cc @@ -263,6 +263,12 @@ INSTANTIATE_TEST_SUITE_P( request.mutable_decoder_experiment_params() ->set_enable_realtime_conversion_v2(true); return request; + }(), + []() { + auto request = GetMobileRequest(); + request.mutable_decoder_experiment_params() + ->set_enable_findability_oriented_order(true); + return request; }()))); TEST_P(SessionHandlerScenarioTestForRequest, TestImplBase) {