diff --git a/python/README.md b/python/README.md index 4d95d7fb..b1ad3e5e 100644 --- a/python/README.md +++ b/python/README.md @@ -66,7 +66,7 @@ $ pip install sudachipy ### Step 2. Get a Dictionary -You can get dictionary as a Python package. It make take a while to download the dictionary file (around 70MB for the `core` edition). +You can get dictionary as a Python package. It may take a while to download the dictionary file (around 70MB for the `core` edition). ```bash $ pip install sudachidict_core @@ -209,7 +209,7 @@ There are three editions of Sudachi Dictionary, namely, `small`, `core`, and `fu SudachiPy uses `sudachidict_core` by default. -Dictionaries are installed as Python packages `sudachidict_small`, `sudachidict_core`, and `sudachidict_full`. +Dictionaries can be installed as Python packages `sudachidict_small`, `sudachidict_core`, and `sudachidict_full`. * [SudachiDict-small · PyPI](https://pypi.org/project/SudachiDict-small/) * [SudachiDict-core · PyPI](https://pypi.org/project/SudachiDict-core/) @@ -234,19 +234,19 @@ $ echo "外国人参政権" | sudachipy -s full ### Dictionary option: Python package -You can specify the dictionary with the `Dicionary()` argument; `config_path` or `dict_type`. +You can specify the dictionary with the `Dicionary()` argument; `config` or `dict`. ```python -class Dictionary(config_path=None, resource_dir=None, dict_type=None) +class Dictionary(config=None, resource_dir=None, dict=None) ``` -1. `config_path` - * You can specify the file path to the setting file with `config_path` (See [Dictionary in The Setting File](#Dictionary in The Setting File) for the detail). +1. `config` + * You can specify the file path to the setting file with `config` (See [Dictionary in The Setting File](#Dictionary in The Setting File) for the detail). * If the dictionary file is specified in the setting file as `systemDict`, SudachiPy will use the dictionary. -2. `dict_type` - * You can also specify the dictionary type with `dict_type`. - * The available arguments are `small`, `core`, or `full`. - * If different dictionaries are specified with `config_path` and `dict_type`, **a dictionary defined `dict_type` overrides** those defined in the config path. +2. `dict` + * You can also specify the dictionary type with `dict`. + * The available arguments are `small`, `core`, `full`, or a path to the dictionary file. + * If different dictionaries are specified with `config` and `dict`, **a dictionary defined `dict` overrides** those defined in the config. ```python from sudachipy import Dictionary @@ -255,16 +255,16 @@ from sudachipy import Dictionary tokenizer_obj = Dictionary().create() # The dictionary given by the `systemDict` key in the config file (/path/to/sudachi.json) will be used -tokenizer_obj = Dictionary(config_path="/path/to/sudachi.json").create() +tokenizer_obj = Dictionary(config="/path/to/sudachi.json").create() -# The dictionary specified by `dict_type` will be set. -tokenizer_obj = Dictionary(dict_type="core").create() # sudachidict_core (same as default) -tokenizer_obj = Dictionary(dict_type="small").create() # sudachidict_small -tokenizer_obj = Dictionary(dict_type="full").create() # sudachidict_full +# The dictionary specified by `dict` will be used. +tokenizer_obj = Dictionary(dict="core").create() # sudachidict_core (same as default) +tokenizer_obj = Dictionary(dict="small").create() # sudachidict_small +tokenizer_obj = Dictionary(dict="full").create() # sudachidict_full -# The dictionary specified by `dict_type` overrides those defined in the config path. +# The dictionary specified by `dict` overrides those defined in the config. # In the following code, `sudachidict_full` will be used regardless of a dictionary defined in the config file. -tokenizer_obj = Dictionary(config_path="/path/to/sudachi.json", dict_type="full").create() +tokenizer_obj = Dictionary(config="/path/to/sudachi.json", dict="full").create() ``` @@ -303,10 +303,8 @@ Then specify your `sudachi.json` with the `-r` option. $ sudachipy -r path/to/sudachi.json ``` - You can build a user dictionary with the subcommand `ubuild`. - ```bash $ sudachipy ubuild -h usage: sudachipy ubuild [-h] [-o file] [-d string] -s file file [file ...] diff --git a/python/py_src/sudachipy/__init__.py b/python/py_src/sudachipy/__init__.py index bdf67f40..fb551538 100644 --- a/python/py_src/sudachipy/__init__.py +++ b/python/py_src/sudachipy/__init__.py @@ -5,6 +5,7 @@ MorphemeList, Morpheme, WordInfo, + PosMatcher, ) from .config import Config from . import errors diff --git a/python/py_src/sudachipy/errors.py b/python/py_src/sudachipy/errors.py index e75e21cd..c11a8205 100644 --- a/python/py_src/sudachipy/errors.py +++ b/python/py_src/sudachipy/errors.py @@ -1,4 +1,4 @@ -# Copyright (c) 2023 Works Applications Co., Ltd. +# Copyright (c) 2023-2024 Works Applications Co., Ltd. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,4 +13,6 @@ # limitations under the License. class SudachiError(Exception): - pass \ No newline at end of file + """Base class for all Sudachipy exceptions. + """ + pass diff --git a/python/py_src/sudachipy/sudachipy.pyi b/python/py_src/sudachipy/sudachipy.pyi index 16c416f6..ca39a95c 100644 --- a/python/py_src/sudachipy/sudachipy.pyi +++ b/python/py_src/sudachipy/sudachipy.pyi @@ -1,6 +1,20 @@ +# Copyright (c) 2024 Works Applications Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. from typing import ClassVar, Iterator, List, Tuple, Union, Callable, Iterable, Optional, Literal, Set from .config import Config +# Part Of Speech POS = Tuple[str, str, str, str, str, str] # POS element PE = Optional[str] @@ -14,10 +28,20 @@ PartialPOS = Union[ Tuple[()], ] +""" +Fields that can be specified for partial dictionary loading. +See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. +""" FieldSet = Optional[Set[Literal["surface", "pos", "normalized_form", "dictionary_form", "reading_form", "word_structure", "split_a", "split_b", "synonym_group_id"]]] +""" +Strings that can be parsed as SplitMode +""" +SplitModeStr = Literal["A", "a", "B", "b", "C", "c"] + + class SplitMode: """ Unit to split text. @@ -34,10 +58,12 @@ class SplitMode: C: ClassVar[SplitMode] = ... @classmethod - def __init__(cls, mode: str = "C") -> None: + def __init__(cls, mode: Optional[SplitModeStr] = "C") -> None: """ - Creates a split mode from a string value - :param mode: string representation of the split mode + Creates a split mode from a string value. + + :param mode: string representation of the split mode. One of [A,B,C] in captital or lower case. + If None, returns SplitMode.C. """ ... @@ -54,14 +80,15 @@ class Dictionary: Creates a sudachi dictionary. If both config.systemDict and dict are not given, `sudachidict_core` is used. - If both config.systemDict and dict are given, dict_type is used. + If both config.systemDict and dict are given, dict is used. + If dict is an absolute path to a file, it is used as a dictionary. - :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.config.Config] object - :param config: alias to config_path, only one of them can be specified at the same time - :param resource_dir: path to the resource directory folder + :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. + :param config: alias to config_path, only one of them can be specified at the same time. + :param resource_dir: path to the resource directory folder. :param dict: type of pre-packaged system dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. Also, can be an _absolute_ path to a compiled dictionary file. - :param dict_type: deprecated alias to dict + :param dict_type: deprecated alias to dict. """ ... @@ -72,16 +99,16 @@ class Dictionary: ... def create(self, - mode: Union[SplitMode, Literal["A", "B", "C"]] = SplitMode.C, - fields: FieldSet = None, + mode: Union[SplitMode, SplitModeStr, None] = SplitMode.C, + fields: Optional[FieldSet] = None, *, - projection: str = None) -> Tokenizer: + projection: Optional[str] = None) -> Tokenizer: """ - Creates a Sudachi Tokenizer. + Creates a sudachi tokenizer. :param mode: sets the analysis mode for this Tokenizer :param fields: load only a subset of fields. - See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html + See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. :param projection: Projection override for created Tokenizer. See Config.projection for values. """ ... @@ -91,32 +118,32 @@ class Dictionary: Creates a new POS matcher. If target is a function, then it must return whether a POS should match or not. - If target a list, it should contain partially specified POS. - By partially specified it means that it is possible to omit POS fields or - use None as a sentinel value that matches any POS. + If target is a list, it should contain partially specified POS. + By partially specified it means that it is possible to omit POS fields or use None as a sentinel value that matches any POS. For example, ('名詞',) will match any noun and (None, None, None, None, None, '終止形') will match any word in 終止形 conjugation form. - :param target: can be either a function or a list of POS tuples. + :param target: can be either a list of POS partial tuples or a callable which maps POS to bool. """ ... def pre_tokenizer(self, - mode: Union[SplitMode, Literal["A", "B", "C"]] = "C", - fields: FieldSet = None, - handler: Optional[Callable[[int, object, MorphemeList], list]] = None, + mode: Union[SplitMode, SplitModeStr, None] = SplitMode.C, + fields: Optional[FieldSet] = None, + handler: Optional[Callable[[ + int, object, MorphemeList], list]] = None, *, - projection: str = None) -> object: + projection: Optional[str] = None) -> object: """ Creates HuggingFace Tokenizers-compatible PreTokenizer. Requires package `tokenizers` to be installed. :param mode: Use this split mode (C by default) - :param fields: ask Sudachi to load only a subset of fields. See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html - :param handler: custom callable to transform MorphemeList into list of tokens. See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py - First two parameters are the index (int) and HuggingFace NormalizedString. - The handler must return a List[NormalizedString]. By default, just segment the tokens. + :param fields: ask Sudachi to load only a subset of fields. See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + :param handler: a custom callable to transform MorphemeList into list of tokens. If None, simply use surface as token representations. + It should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. + See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py. :param projection: Projection override for created Tokenizer. See Config.projection for values. """ ... @@ -126,7 +153,7 @@ class Dictionary: Returns POS with the given id. :param pos_id: POS id - :return: POS tuple with the given id. + :return: POS tuple with the given id or None for non existing id. """ ... @@ -180,6 +207,9 @@ class Morpheme: def get_word_info(self) -> WordInfo: """ Returns the word info. + + ..deprecated:: v0.6.0 + Users should not touch the raw WordInfo. """ ... @@ -197,7 +227,8 @@ class Morpheme: def part_of_speech(self) -> POS: """ - Returns the part of speech. + Returns the part of speech as a six-element tuple. + Tuple elements are four POS levels, conjugation type and conjugation form. """ ... @@ -213,12 +244,15 @@ class Morpheme: """ ... - def split(self, mode: Union[SplitMode, Literal["A", "B", "C"]], out: Optional[MorphemeList] = None, add_single: bool = True) -> MorphemeList: + def split(self, + mode: Union[SplitMode, SplitModeStr], + out: Optional[MorphemeList] = None, + add_single: bool = True) -> MorphemeList: """ Returns sub-morphemes in the provided split mode. - :param mode: mode of new split - :param out: write results to this MorhpemeList instead of creating new one + :param mode: mode of new split. + :param out: write results to this MorhpemeList instead of creating new one. See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for more information on output parameters. Returned MorphemeList will be invalidated if this MorphemeList is used as an output parameter. @@ -230,6 +264,7 @@ class Morpheme: def surface(self) -> str: """ Returns the substring of input text corresponding to the morpheme, or a projection if one is configured. + See `Config.projection`. """ ... @@ -237,6 +272,7 @@ class Morpheme: def raw_surface(self) -> str: """ Returns the substring of input text corresponding to the morpheme regardless the configured projection. + See `Config.projection`. """ ... @@ -255,13 +291,14 @@ class Morpheme: def __len__(self) -> int: """ - Returns morpheme length in codepoints + Returns morpheme length in codepoints. """ class MorphemeList: """ A list of morphemes. + An object can not be instantiated manually. Use Tokenizer.tokenize("") to create an empty morpheme list. """ @@ -269,9 +306,12 @@ class MorphemeList: def __init__(self) -> None: ... @classmethod - def empty(cls, dict) -> MorphemeList: + def empty(cls, dict: Dictionary) -> MorphemeList: """ Returns an empty morpheme list with dictionary. + + .. deprecated:: + Use Tokenizer.tokenize("") if you need. """ ... @@ -287,29 +327,35 @@ class MorphemeList: """ ... - def __getitem__(self, index) -> Morpheme: ... + def __getitem__(self, index: int) -> Morpheme: ... def __iter__(self) -> Iterator[Morpheme]: ... def __len__(self) -> int: ... class Tokenizer: + """ + A sudachi tokenizer + + Create using Dictionary.create method. + """ SplitMode: ClassVar[SplitMode] = ... + @classmethod def __init__(cls) -> None: ... - def tokenize(self, text: str, - mode: Union[SplitMode, Literal["A", "B", "C"]] = ..., + def tokenize(self, + text: str, + mode: Union[SplitMode, SplitModeStr, None] = None, out: Optional[MorphemeList] = None) -> MorphemeList: """ Break text into morphemes. - SudachiPy 0.5.* had logger parameter, it is accepted, but ignored. - - :param text: text to analyze + :param text: text to analyze. :param mode: analysis mode. This parameter is deprecated. Pass the analysis mode at the Tokenizer creation time and create different tokenizers for different modes. If you need multi-level splitting, prefer using :py:meth:`Morpheme.split` method instead. + :param logger: Arg for v0.5.* compatibility. Ignored. :param out: tokenization results will be written into this MorphemeList, a new one will be created instead. See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. """ @@ -336,47 +382,51 @@ class WordInfo: surface: ClassVar[str] = ... synonym_group_ids: ClassVar[List[int]] = ... word_structure: ClassVar[List[int]] = ... + @classmethod def __init__(self) -> None: ... def length(self) -> int: ... class PosMatcher: + """ + A part-of-speech matcher which checks if a morpheme belongs to a set of part of speech. + + Create using Dictionary.pos_matcher method. + """ + def __iter__(self) -> Iterator[POS]: ... def __len__(self) -> int: ... - def __call__(self, m: Morpheme) -> bool: + def __call__(self, /, m: Morpheme) -> bool: """ - Checks whether a morpheme has matching POS - :param m: morpheme - :return: if morpheme has matching POS + Checks whether a morpheme has matching POS. + + :param m: a morpheme to check. + :return: if morpheme has matching POS. """ ... def __or__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if any of two matchers would match it - :return: PosMatcher + Returns a POS matcher which matches a POS if any of two matchers would match it. """ ... def __and__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if both matchers would match it at the same time - :return: PosMatcher + Returns a POS matcher which matches a POS if both matchers would match it at the same time. """ ... def __sub__(self, other: PosMatcher) -> PosMatcher: """ - Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS - :return: PosMatcher + Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS. """ ... def __invert__(self) -> PosMatcher: """ - Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher - :return: PosMatcher + Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher. """ ... diff --git a/python/src/build.rs b/python/src/build.rs index 40e52c34..83e3dfb5 100644 --- a/python/src/build.rs +++ b/python/src/build.rs @@ -58,8 +58,20 @@ fn create_file(p: &Path) -> std::io::Result { OpenOptions::new().create_new(true).write(true).open(p) } +/// Build system dictionary from matrix and lexicons. +/// +/// :param matrix: Path to the matrix file. +/// :param lex: List of paths to lexicon files. +/// :param output: Path to output built dictionray. +/// :param description: A description text to embed in the dictionary. +/// :return: A build report, list of (part, size, time). +/// +/// :type matrix: pathlib.Path | str | bytes +/// :type lex: list[pathlib.Path | str | bytes] +/// :type output: pathlib.Path | str +/// :type description: str #[pyfunction] -#[pyo3(text_signature = "(matrix, lex, output, description=None) -> list")] +#[pyo3(text_signature = "(matrix, lex, output, description=None) -> list[tuple[str, int, float]]")] fn build_system_dic<'py>( py: Python<'py>, matrix: &Bound<'py, PyAny>, @@ -90,8 +102,20 @@ fn build_system_dic<'py>( to_stats(py, builder) } +/// Build user dictionary from lexicons based on the given system dictionary. +/// +/// :param system: Path to the system dictionary. +/// :param lex: List of paths to lexicon files. +/// :param output: Path to output built dictionray. +/// :param description: A description text to embed in the dictionary. +/// :return: A build report, list of (part, size, time). +/// +/// :type system: pathlib.Path | str +/// :type lex: list[pathlib.Path | str | bytes] +/// :type output: pathlib.Path | str +/// :type description: str #[pyfunction] -#[pyo3(text_signature = "(system, lex, output, description=None) -> list")] +#[pyo3(text_signature = "(system, lex, output, description=None) -> list[tuple[str, int, float]]")] fn build_user_dic<'py>( py: Python<'py>, system: &Bound<'py, PyAny>, diff --git a/python/src/dictionary.rs b/python/src/dictionary.rs index e9cbf1ed..b5d6dc1b 100644 --- a/python/src/dictionary.rs +++ b/python/src/dictionary.rs @@ -78,7 +78,24 @@ impl PyDicData { } } -/// A sudachi dictionary +/// A sudachi dictionary. +/// +/// If both config.systemDict and dict are not given, `sudachidict_core` is used. +/// If both config.systemDict and dict are given, dict is used. +/// If dict is an absolute path to a file, it is used as a dictionary. +/// +/// :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. +/// :param config: alias to config_path, only one of them can be specified at the same time. +/// :param resource_dir: path to the resource directory folder. +/// :param dict: type of pre-packaged dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. +/// Also, can be an _absolute_ path to a compiled dictionary file. +/// :param dict_type: deprecated alias to dict. +/// +/// :type config_path: Config | pathlib.Path | str | None +/// :type config: Config | pathlib.Path | str | None +/// :type resource_dir: pathlib.Path | str | None +/// :type dict: pathlib.Path | str | None +/// :type dict_type: pathlib.Path | str | None #[pyclass(module = "sudachipy.dictionary", name = "Dictionary")] #[derive(Clone)] pub struct PyDictionary { @@ -90,17 +107,27 @@ pub struct PyDictionary { impl PyDictionary { /// Creates a sudachi dictionary. /// - /// If both config.systemDict and dict_type are not given, `sudachidict_core` is used. - /// If both config.systemDict and dict_type are given, dict_type is used. - /// If dict is an absolute path to a file, it is used as a dictionary + /// If both config.systemDict and dict are not given, `sudachidict_core` is used. + /// If both config.systemDict and dict are given, dict is used. + /// If dict is an absolute path to a file, it is used as a dictionary. /// - /// :param config_path: path to the configuration JSON file - /// :param resource_dir: path to the resource directory folder - /// :param dict: type of pre-packaged dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. + /// :param config_path: path to the configuration JSON file, config json as a string, or a [sudachipy.Config] object. + /// :param config: alias to config_path, only one of them can be specified at the same time. + /// :param resource_dir: path to the resource directory folder. + /// :param dict: type of pre-packaged dictionary, referring to sudachidict_ packages on PyPI: https://pypi.org/search/?q=sudachidict. /// Also, can be an _absolute_ path to a compiled dictionary file. - /// :param dict_type: deprecated alias to dict + /// :param dict_type: deprecated alias to dict. + /// + /// :type config_path: Config | pathlib.Path | str | None + /// :type config: Config | pathlib.Path | str | None + /// :type resource_dir: pathlib.Path | str | None + /// :type dict: pathlib.Path | str | None + /// :type dict_type: pathlib.Path | str | None #[new] - #[pyo3(signature=(config_path = None, resource_dir = None, dict = None, dict_type = None, *, config = None))] + #[pyo3( + text_signature="(config_path=None, resource_dir=None, dict=None, dict_type=None, *, config=None) -> Dictionary", + signature=(config_path=None, resource_dir=None, dict=None, dict_type=None, *, config=None) + )] fn new( py: Python, config_path: Option<&Bound>, @@ -216,12 +243,17 @@ impl PyDictionary { /// Creates a sudachi tokenizer. /// - /// :param mode: tokenizer's default split mode (C by default). + /// :param mode: sets the analysis mode for this Tokenizer /// :param fields: load only a subset of fields. - /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html + /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + /// :param projection: Projection override for created Tokenizer. See Config.projection for values. + /// + /// :type mode: SplitMode | str | None + /// :type fields: set[str] | None + /// :type projection: str | None #[pyo3( - text_signature = "($self, mode = 'C') -> sudachipy.Tokenizer", - signature = (mode = None, fields = None, *, projection = None) + text_signature="(self, /, mode=SplitMode.C, fields=None, *, projection=None) -> Tokenizer", + signature=(mode=None, fields=None, *, projection=None) )] fn create<'py>( &'py self, @@ -254,15 +286,15 @@ impl PyDictionary { /// Creates a POS matcher object /// /// If target is a function, then it must return whether a POS should match or not. - /// If target a list, it should contain partially specified POS. - /// By partially specified it means that it is possible to omit POS fields or - /// use None as a sentinel value that matches any POS. + /// If target is a list, it should contain partially specified POS. + /// By partially specified it means that it is possible to omit POS fields or use None as a sentinel value that matches any POS. /// /// For example, ('名詞',) will match any noun and /// (None, None, None, None, None, '終止形‐一般') will match any word in 終止形‐一般 conjugation form. /// - /// :param target: can be either a callable or list of POS partial tuples - #[pyo3(text_signature = "($self, target)")] + /// :param target: can be either a list of POS partial tuples or a callable which maps POS to bool. + /// + /// :type target: Iterable[PartialPOS] | Callable[[POS], bool] fn pos_matcher<'py>( &'py self, py: Python<'py>, @@ -276,19 +308,19 @@ impl PyDictionary { /// /// :param mode: Use this split mode (C by default) /// :param fields: ask Sudachi to load only a subset of fields. - /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html - /// :param handler: a custom callable to transform MorphemeList into list of tokens. - /// It should be should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. - /// See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py - /// If nothing was passed, simply use surface as token representations. - /// :param projection: projection mode for a created PreTokenizer. - /// See :class:`sudachipy.config.Config` object documentation for supported projections. + /// See https://worksapplications.github.io/sudachi.rs/python/topics/subsetting.html. + /// :param handler: a custom callable to transform MorphemeList into list of tokens. If None, simply use surface as token representations. + /// It should be a `function(index: int, original: NormalizedString, morphemes: MorphemeList) -> List[NormalizedString]`. + /// See https://github.com/huggingface/tokenizers/blob/master/bindings/python/examples/custom_components.py. + /// :param projection: Projection override for created Tokenizer. See Config.projection for values. /// - /// :type mode: sudachipy.SplitMode - /// :type fields: Set[str] + /// :type mode: SplitMode | str | None + /// :type fields: set[str] | None + /// :type handler: Callable[[int, NormalizedString, MorphemeList], list[NormalizedString]] | None + /// :type projection: str | None #[pyo3( - text_signature = "($self, mode, fields, handler) -> tokenizers.PreTokenizer", - signature = (mode = None, fields = None, handler = None, *, projection = None) + text_signature="(self, /, mode=None, fields=None, handler=None, *, projection=None) -> tokenizers.PreTokenizer", + signature=(mode=None, fields=None, handler=None, *, projection=None) )] fn pre_tokenizer<'py>( &'py self, @@ -341,9 +373,10 @@ impl PyDictionary { /// :param surface: find all morphemes with the given surface /// :param out: if passed, reuse the given morpheme list instead of creating a new one. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. + /// /// :type surface: str - /// :type out: sudachipy.MorphemeList - #[pyo3(text_signature = "($self, surface, out = None) -> sudachipy.MorphemeList")] + /// :type out: MorphemeList | None + #[pyo3(text_signature = "(self, /, surface, out=None) -> MorphemeList")] fn lookup<'py>( &'py self, py: Python<'py>, @@ -370,14 +403,19 @@ impl PyDictionary { Ok(l) } - /// Close this dictionary - #[pyo3(text_signature = "($self)")] + /// Close this dictionary. + #[pyo3(text_signature = "(self, /) -> ()")] fn close(&mut self) { self.dictionary = None; } - /// Get POS Tuple by its id - #[pyo3(text_signature = "($self, pos_id: int)")] + /// Returns POS with the given id. + /// + /// :param pos_id: POS id + /// :return: POS tuple with the given id or None for non existing id. + /// + /// :type pos_id: int + #[pyo3(text_signature = "(self, /, pos_id: int) -> tuple[str, str, str, str, str, str] | None")] fn pos_of<'py>(&'py self, py: Python<'py>, pos_id: usize) -> Option<&Bound<'py, PyTuple>> { let dic = self.dictionary.as_ref().unwrap(); dic.pos.get(pos_id).map(|x| x.bind(py)) diff --git a/python/src/lib.rs b/python/src/lib.rs index f2c13703..3a5f8f84 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -26,7 +26,9 @@ mod projection; mod tokenizer; mod word_info; -/// module root +/// SudachiPy raw module root. +/// +/// Users should not use this directly. #[pymodule] fn sudachipy(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; @@ -35,6 +37,7 @@ fn sudachipy(_py: Python, m: &Bound) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; build::register_functions(m)?; Ok(()) } diff --git a/python/src/morpheme.rs b/python/src/morpheme.rs index 69418d32..1321ea4c 100644 --- a/python/src/morpheme.rs +++ b/python/src/morpheme.rs @@ -31,7 +31,10 @@ use crate::word_info::PyWordInfo; pub(crate) type PyMorphemeList = MorphemeList>; pub(crate) type PyProjector = Option>; -/// A list of morphemes +/// A list of morphemes. +/// +/// An object can not be instantiated manually. +/// Use Tokenizer.tokenize("") to create an empty morpheme list. #[pyclass(module = "sudachipy.morphemelist", name = "MorphemeList")] pub struct PyMorphemeListWrapper { /// use `internal()` function instead @@ -86,11 +89,15 @@ impl PyMorphemeListWrapper { } } } + #[pymethods] impl PyMorphemeListWrapper { - /// Returns an empty morpheme list with dictionary + /// Returns an empty morpheme list with dictionary. + /// + /// .. deprecated:: 0.6.0 + /// Use Tokenizer.tokenize("") if you need. #[classmethod] - #[pyo3(text_signature = "(dict: sudachipy.Dictionary) -> sudachipy.MorphemeList")] + #[pyo3(text_signature = "(dict: Dictionary) -> MorphemeList")] fn empty(_cls: &Bound, py: Python, dict: &PyDictionary) -> PyResult { let cat = PyModule::import_bound(py, "builtins")?.getattr("DeprecationWarning")?; PyErr::warn_bound( @@ -108,14 +115,14 @@ impl PyMorphemeListWrapper { }) } - /// Returns the total cost of the path - #[pyo3(text_signature = "($self) -> int")] + /// Returns the total cost of the path. + #[pyo3(text_signature = "(self, /) -> int")] fn get_internal_cost(&self, py: Python) -> i32 { self.internal(py).get_internal_cost() } /// Returns the number of morpheme in this list. - #[pyo3(text_signature = "($self) -> int")] + #[pyo3(text_signature = "(self, /) -> int")] fn size(&self, py: Python) -> usize { self.internal(py).len() } @@ -197,7 +204,7 @@ impl PyMorphemeListWrapper { } } -/// A morpheme (basic semantic unit of language). +/// An iterator over the MorphemeList. #[pyclass(module = "sudachipy.morphemelist", name = "MorphemeIter")] pub struct PyMorphemeIter { list: Py, @@ -241,6 +248,7 @@ impl<'py> Deref for MorphemeRef<'py> { } } +/// A morpheme (basic semantic unit of language). #[pyclass(module = "sudachipy.morpheme", name = "Morpheme", frozen)] pub struct PyMorpheme { list: Py, @@ -276,22 +284,24 @@ impl PyMorpheme { #[pymethods] impl PyMorpheme { - /// Returns the begin index of this in the input text - #[pyo3(text_signature = "($self) -> int")] + /// Returns the begin index of this in the input text. + #[pyo3(text_signature = "(self, /) -> int")] fn begin(&self, py: Python) -> usize { // call codepoint version self.morph(py).begin_c() } - /// Returns the end index of this in the input text - #[pyo3(text_signature = "($self) -> int")] + /// Returns the end index of this in the input text. + #[pyo3(text_signature = "(self, /) -> int")] fn end(&self, py: Python) -> usize { // call codepoint version self.morph(py).end_c() } - /// Returns the substring of input text corresponding to the morpheme, or a projection if one is configured - #[pyo3(text_signature = "($self) -> str")] + /// Returns the substring of input text corresponding to the morpheme, or a projection if one is configured. + /// + /// See `Config.projection`. + #[pyo3(text_signature = "(self, /) -> str")] fn surface<'py>(&'py self, py: Python<'py>) -> Bound<'py, PyString> { let list = self.list(py); let morph = self.morph(py); @@ -301,15 +311,17 @@ impl PyMorpheme { } } - /// Returns the substring of input text corresponding to the morpheme regardless the configured projection - #[pyo3(text_signature = "($self) -> str")] + /// Returns the substring of input text corresponding to the morpheme regardless the configured projection. + /// + /// See `Config.projection`. + #[pyo3(text_signature = "(self, /) -> str")] fn raw_surface<'py>(&'py self, py: Python<'py>) -> Bound<'py, PyString> { PyString::new_bound(py, self.morph(py).surface().deref()) } /// Returns the part of speech as a six-element tuple. - /// Tuple elements are four POS levels, conjugation type and conjugation form. - #[pyo3(text_signature = "($self)")] + /// Tuple elements are four POS levels, conjugation type and conjugation form. + #[pyo3(text_signature = "(self, /) -> tuple[str, str, str, str, str, str]")] fn part_of_speech<'py>(&'py self, py: Python<'py>) -> Py { let pos_id = self.part_of_speech_id(py); self.list(py) @@ -319,45 +331,44 @@ impl PyMorpheme { .clone_ref(py) } - /// Returns the id of the part of speech in the dictionary - #[pyo3(text_signature = "($self) -> int")] + /// Returns the id of the part of speech in the dictionary. + #[pyo3(text_signature = "(self, /) -> int")] pub fn part_of_speech_id(&self, py: Python) -> u16 { self.morph(py).part_of_speech_id() } - /// Returns the dictionary form - #[pyo3(text_signature = "($self) -> str")] + /// Returns the dictionary form. + #[pyo3(text_signature = "(self, /) -> str")] fn dictionary_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().dictionary_form().into_py(py) } - /// Returns the normalized form - #[pyo3(text_signature = "($self) -> str")] + /// Returns the normalized form. + #[pyo3(text_signature = "(self, /) -> str")] fn normalized_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().normalized_form().into_py(py) } - /// Returns the reading form - #[pyo3(text_signature = "($self) -> str")] + /// Returns the reading form. + #[pyo3(text_signature = "(self, /) -> str")] fn reading_form<'py>(&'py self, py: Python<'py>) -> PyObject { self.morph(py).get_word_info().reading_form().into_py(py) } /// Returns sub-morphemes in the provided split mode. /// - /// :param mode: mode of new split - /// :param out: write results to this MorhpemeList instead of creating new one + /// :param mode: mode of new split. + /// :param out: write results to this MorhpemeList instead of creating new one. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for /// more information on output parameters. /// Returned MorphemeList will be invalidated if this MorphemeList is used as an output parameter. /// :param add_single: return lists with the current morpheme if the split hasn't produced any elements. /// When False is passed, empty lists are returned instead. - /// :type mode: sudachipy.SplitMode - /// :type out: Optional[sudachipy.MorphemeList] + /// + /// :type mode: SplitMode | None + /// :type out: MorphemeList | None /// :type add_single: bool - #[pyo3( - text_signature = "($self, mode, out = None, add_single = False) -> sudachipy.MorphemeList" - )] + #[pyo3(text_signature = "(self, /, mode, out=None, add_single=False) -> MorphemeList")] fn split<'py>( &'py self, py: Python<'py>, @@ -399,20 +410,20 @@ impl PyMorpheme { Ok(out_cell) } - /// Returns whether if this is out of vocabulary word - #[pyo3(text_signature = "($self) -> bool")] + /// Returns whether if this is out of vocabulary word. + #[pyo3(text_signature = "(self, /) -> bool")] fn is_oov(&self, py: Python) -> bool { self.morph(py).is_oov() } - /// Returns word id of this word in the dictionary - #[pyo3(text_signature = "($self) -> int")] + /// Returns word id of this word in the dictionary. + #[pyo3(text_signature = "(self, /) -> int")] fn word_id(&self, py: Python) -> u32 { self.morph(py).word_id().as_raw() } - /// Returns the dictionary id which this word belongs - #[pyo3(text_signature = "($self) -> int")] + /// Returns the dictionary id which this word belongs. + #[pyo3(text_signature = "(self, /) -> int")] fn dictionary_id(&self, py: Python) -> i32 { let word_id = self.morph(py).word_id(); if word_id.is_oov() { @@ -422,16 +433,19 @@ impl PyMorpheme { } } - /// Returns the list of synonym group ids - #[pyo3(text_signature = "($self) -> List[int]")] + /// Returns the list of synonym group ids. + #[pyo3(text_signature = "(self, /) -> List[int]")] fn synonym_group_ids<'py>(&'py self, py: Python<'py>) -> Bound { let mref = self.morph(py); let ids = mref.get_word_info().synonym_group_ids(); PyList::new_bound(py, ids) } - /// Returns the word info - #[pyo3(text_signature = "($self) -> sudachipy.WordInfo")] + /// Returns the word info. + /// + /// ..deprecated:: v0.6.0 + /// Users should not touch the raw WordInfo. + #[pyo3(text_signature = "(self, /) -> WordInfo")] fn get_word_info(&self, py: Python) -> PyResult { let cat = PyModule::import_bound(py, "builtins")?.getattr("DeprecationWarning")?; PyErr::warn_bound(py, &cat, "Users should not touch the raw WordInfo.", 1)?; @@ -439,7 +453,7 @@ impl PyMorpheme { Ok(self.morph(py).get_word_info().clone().into()) } - /// Returns morpheme length in codepoints + /// Returns morpheme length in codepoints. pub fn __len__(&self, py: Python) -> usize { let m = self.morph(py); m.end_c() - m.begin_c() diff --git a/python/src/pos_matcher.rs b/python/src/pos_matcher.rs index f0753f4b..488d3c1d 100644 --- a/python/src/pos_matcher.rs +++ b/python/src/pos_matcher.rs @@ -26,7 +26,12 @@ use sudachi::pos::PosMatcher; use crate::dictionary::PyDicData; use crate::morpheme::PyMorpheme; -#[pyclass(name = "PosMatcher", module = "sudachipy")] +/// A part-of-speech matcher which checks if a morpheme belongs to a set of part of speech. +/// +/// Create using Dictionary.pos_matcher method. +/// +/// Use `__call__(m: Morpheme) -> bool` to check whether a morpheme has matching POS. +#[pyclass(module = "sudachipy.pos_matcher", name = "PosMatcher")] pub struct PyPosMatcher { matcher: PosMatcher, dic: Arc, @@ -123,6 +128,12 @@ impl PyPosMatcher { #[pymethods] impl PyPosMatcher { + /// Checks whether a morpheme has matching POS. + /// + /// :param m: a morpheme to check. + /// :return: if morpheme has matching POS. + /// + /// :type m: Morpheme pub fn __call__<'py>(&'py self, py: Python<'py>, m: &'py PyMorpheme) -> bool { let pos_id = m.part_of_speech_id(py); self.matcher.matches_id(pos_id) @@ -140,6 +151,7 @@ impl PyPosMatcher { self.matcher.num_entries() } + /// Returns a POS matcher which matches a POS if any of two matchers would match it. pub fn __or__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -153,6 +165,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches a POS if both matchers would match it at the same time. pub fn __and__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -166,6 +179,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches a POS if self would match the POS and other would not match the POS. pub fn __sub__(&self, other: &Self) -> Self { assert_eq!( Arc::as_ptr(&self.dic), @@ -179,6 +193,7 @@ impl PyPosMatcher { } } + /// Returns a POS matcher which matches all POS tags except ones defined in the current POS matcher. pub fn __invert__(&self) -> Self { let max_id = self.dic.pos.len(); // map -> filter chain is needed to handle exactly u16::MAX POS entries @@ -194,7 +209,8 @@ impl PyPosMatcher { } } -#[pyclass(name = "PosMatcherIterator", module = "sudachipy")] +/// An iterator over POS tuples in the PosPatcher +#[pyclass(module = "sudachipy.pos_matcher", name = "PosMatcherIterator")] pub struct PyPosIter { data: Vec, dic: Arc, diff --git a/python/src/pretokenizer.rs b/python/src/pretokenizer.rs index 20e5cf65..7bb9f6c3 100644 --- a/python/src/pretokenizer.rs +++ b/python/src/pretokenizer.rs @@ -76,9 +76,10 @@ impl PerThreadPreTokenizer { } } -/// Binding for the Tokenizer, which handles threading for tokenization +/// Binding for the Tokenizer, which handles threading for tokenization. /// -/// We use ThreadLocal for storing actual tokenizers +/// Create using Dictionary.pre_tokenizer method. +/// We use ThreadLocal for storing actual tokenizers. #[pyclass(module = "sudachipy.pretokenizer", name = "SudachiPreTokenizer")] pub struct PyPretokenizer { dict: Arc, diff --git a/python/src/tokenizer.rs b/python/src/tokenizer.rs index cc8142e7..ba1cb632 100644 --- a/python/src/tokenizer.rs +++ b/python/src/tokenizer.rs @@ -29,14 +29,18 @@ use crate::dictionary::{extract_mode, PyDicData}; use crate::errors::SudachiError as SudachiPyErr; use crate::morpheme::{PyMorphemeListWrapper, PyProjector}; -/// Unit to split text +/// Unit to split text. /// /// A == short mode /// /// B == middle mode /// /// C == long mode -// +/// +/// :param mode: string representation of the split mode. One of [A,B,C] in captital or lower case. +/// If None, returns SplitMode.C. +/// +/// :type mode: str | None #[pyclass(module = "sudachipy.tokenizer", name = "SplitMode", frozen)] #[derive(Clone, PartialEq, Eq, Copy, Debug)] #[repr(u8)] @@ -68,7 +72,17 @@ impl From for PySplitMode { #[pymethods] impl PySplitMode { + /// Creates a split mode from a string value. + /// + /// :param mode: string representation of the split mode. One of [A,B,C] in captital or lower case. + /// If None, returns SplitMode.C. + /// + /// :type mode: str | None #[new] + #[pyo3( + text_signature="(mode=None) -> SplitMode", + signature=(mode=None) + )] fn new(mode: Option<&str>) -> PyResult { let mode = match mode { Some(m) => m, @@ -82,7 +96,9 @@ impl PySplitMode { } } -/// Sudachi Tokenizer, Python version +/// A sudachi tokenizer +/// +/// Create using Dictionary.create method. #[pyclass(module = "sudachipy.tokenizer", name = "Tokenizer")] pub(crate) struct PyTokenizer { tokenizer: StatefulTokenizer>, @@ -115,21 +131,21 @@ impl PyTokenizer { /// Break text into morphemes. /// - /// SudachiPy 0.5.* had logger parameter, it is accepted, but ignored. - /// - /// :param text: text to analyze + /// :param text: text to analyze. /// :param mode: analysis mode. /// This parameter is deprecated. /// Pass the analysis mode at the Tokenizer creation time and create different tokenizers for different modes. /// If you need multi-level splitting, prefer using :py:meth:`Morpheme.split` method instead. + /// :param logger: Arg for v0.5.* compatibility. Ignored. /// :param out: tokenization results will be written into this MorphemeList, a new one will be created instead. /// See https://worksapplications.github.io/sudachi.rs/python/topics/out_param.html for details. + /// /// :type text: str - /// :type mode: sudachipy.SplitMode - /// :type out: sudachipy.MorphemeList + /// :type mode: SplitMode | str | None + /// :type out: MorphemeList #[pyo3( - text_signature = "($self, text: str, mode = None, logger = None, out = None) -> sudachipy.MorphemeList", - signature = (text, mode = None, logger = None, out = None) + text_signature="(self, /, text: str, mode=None, logger=None, out=None) -> MorphemeList", + signature=(text, mode=None, logger=None, out=None) )] #[allow(unused_variables)] fn tokenize<'py>( @@ -182,6 +198,7 @@ impl PyTokenizer { Ok(out_list) } + /// SplitMode of the tokenizer. #[getter] fn mode(&self) -> PySplitMode { self.tokenizer.mode().into()