"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " [2272/2272 20:54, Epoch 4/4]\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | Epoch | \n",
+ " Training Loss | \n",
+ " Validation Loss | \n",
+ " Bleu | \n",
+ " Gen Len | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 1 | \n",
+ " 0.941200 | \n",
+ " 0.653884 | \n",
+ " 67.599700 | \n",
+ " 127.287700 | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " 0.697700 | \n",
+ " 0.608778 | \n",
+ " 69.201200 | \n",
+ " 127.783500 | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " 0.631200 | \n",
+ " 0.589383 | \n",
+ " 69.907900 | \n",
+ " 127.373200 | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " 0.587600 | \n",
+ " 0.584227 | \n",
+ " 70.363300 | \n",
+ " 126.859000 | \n",
+ "
\n",
+ " \n",
+ "
"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/plain": [
+ "TrainOutput(global_step=2272, training_loss=0.6974154055958063, metrics={'train_runtime': 1218.4183, 'train_samples_per_second': 29.816, 'train_steps_per_second': 1.865, 'total_flos': 2196400063905792.0, 'train_loss': 0.6974154055958063, 'epoch': 4.0})"
+ ]
+ },
+ "execution_count": 26,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "trainer.train()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 27,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ " \n",
+ " \n",
+ "
\n",
+ " [44/44 06:58]\n",
+ "
Run history:
| eval/bleu | ▂▅▇██▁ |
| eval/gen_len | ▅█▅▂▂▁ |
| eval/loss | █▃▂▁▁█ |
| eval/runtime | ██▇▇▇▁ |
| eval/samples_per_second | ▁▁▆███ |
| eval/steps_per_second | ▁▁▅▆▆█ |
| train/epoch | ▁▁▃▄▅▆▇████ |
| train/global_step | ▁▁▃▄▅▆▇████ |
| train/learning_rate | █▆▃▁ |
| train/loss | █▃▂▁ |
| train/total_flos | ▁ |
| train/train_loss | ▁ |
| train/train_runtime | ▁ |
| train/train_samples_per_second | ▁ |
| train/train_steps_per_second | ▁ |
Run summary:
| eval/bleu | 67.197 |
| eval/gen_len | 126.7394 |
| eval/loss | 0.65617 |
| eval/runtime | 194.0045 |
| eval/samples_per_second | 2.928 |
| eval/steps_per_second | 0.186 |
| train/epoch | 4.0 |
| train/global_step | 2272 |
| train/learning_rate | 0.0 |
| train/loss | 0.5876 |
| train/total_flos | 2196400063905792.0 |
| train/train_loss | 0.69742 |
| train/train_runtime | 1218.4183 |
| train/train_samples_per_second | 29.816 |
| train/train_steps_per_second | 1.865 |
View job at https://wandb.ai/wide-learning/huggingface/jobs/QXJ0aWZhY3RDb2xsZWN0aW9uOjExNjYyNzQ4OA==/version_details/v0
Synced 5 W&B file(s), 0 media file(s), 11 artifact file(s) and 0 other file(s)"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "data": {
+ "text/html": [
+ "Find logs at: ./wandb/run-20231117_130518-fgt5hc0j/logs"
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "wandb.finish()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "for dirname, _, filenames in os.walk(model_name):\n",
+ " for filename in filenames:\n",
+ " print(os.path.join(dirname, filename))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "44e967b87fa3423a80769bccdb3972e9",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Downloading (…)okenizer_config.json: 0%| | 0.00/816 [00:00=0.5.1
openai
tqdm
pandas
+
+# RodionfromHSE
+accelerate==0.24.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+antlr4-python3-runtime==4.9.3
+appnope==0.1.3
+asttokens==2.4.0
+async-timeout==4.0.3
+attrs==23.1.0
+backcall==0.2.0
+certifi==2023.7.22
+charset-normalizer==3.3.1
+colorama==0.4.6
+comm==0.1.4
+datasets==2.14.6
+debugpy==1.8.0
+decorator==5.1.1
+dill==0.3.7
+executing==2.0.0
+filelock==3.12.4
+frozenlist==1.4.0
+fsspec==2023.10.0
+huggingface-hub==0.17.3
+hydra-core==1.3.2
+idna==3.4
+ipykernel==6.26.0
+ipython==8.16.1
+ipywidgets==8.1.1
+jedi==0.19.1
+Jinja2==3.1.2
+jupyter_client==8.5.0
+jupyter_core==5.4.0
+jupyterlab-widgets==3.0.9
+lxml==4.9.3
+MarkupSafe==2.1.3
+matplotlib-inline==0.1.6
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.15
+nest-asyncio==1.5.8
+networkx==3.2.1
+numpy==1.26.1
+omegaconf==2.3.0
+packaging==23.2
+pandas==2.1.1
+parso==0.8.3
+pexpect==4.8.0
+pickleshare==0.7.5
+platformdirs==3.11.0
+portalocker==2.8.2
+prompt-toolkit==3.0.39
+protobuf==4.25.0
+psutil==5.9.6
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==13.0.0
+Pygments==2.16.1
+python-dateutil==2.8.2
+pytz==2023.3.post1
+PyYAML==6.0.1
+pyzmq==25.1.1
+regex==2023.10.3
+requests==2.31.0
+sacrebleu==2.3.2
+safetensors==0.4.0
+sentencepiece==0.1.99
+six==1.16.0
+stack-data==0.6.3
+sympy==1.12
+tabulate==0.9.0
+tokenizers==0.14.1
+torch==2.1.0
+tornado==6.3.3
+tqdm==4.66.1
+traitlets==5.12.0
+transformers==4.34.1
+typing_extensions==4.8.0
+tzdata==2023.3
+urllib3==2.0.7
+wcwidth==0.2.8
+widgetsnbextension==4.0.9
+xxhash==3.4.1
+yarl==1.9.2
+zstandard==0.21.0
diff --git a/src/finetune/eval_bleu.py b/src/finetune/eval_bleu.py
new file mode 100644
index 0000000..b358625
--- /dev/null
+++ b/src/finetune/eval_bleu.py
@@ -0,0 +1,33 @@
+import os
+import sys
+
+root_dir = os.path.abspath(os.path.join(__file__, '../../..'))
+sys.path.append(root_dir)
+
+import pandas as pd
+import sacrebleu
+from omegaconf import OmegaConf
+from custom_utils.config_handler import read_config, pprint_config
+
+
+def main() -> None:
+ cfg: OmegaConf = read_config(overrides=["dataset=model_eval"])
+ pprint_config(cfg)
+
+ # Load data
+ data = pd.read_csv(cfg.dataset.path)
+
+ # compute blue for each candidate
+ cols = cfg.dataset.cols
+ reference_col = cols.reference
+ data[reference_col] = data[reference_col].apply(lambda x: [x])
+ for candidate_col in cols.candidates:
+ bleu = data[[reference_col, candidate_col]].apply(
+ lambda x: sacrebleu.corpus_bleu(x[reference_col], x[candidate_col]).score,
+ axis=1
+ )
+ data[f"{candidate_col}_bleu"] = bleu
+ print(f"{candidate_col} BLEU: {bleu.mean():.2f}")
+
+if __name__ == "__main__":
+ main()
diff --git a/src/prod/features/__init__.py b/src/prod/features/__init__.py
new file mode 100644
index 0000000..f14d04c
--- /dev/null
+++ b/src/prod/features/__init__.py
@@ -0,0 +1,3 @@
+from .attention_extractor import BaseAttentionExtractor, choose_attention_extractor
+from .synonym_searcher import BaseSynonymSearcher, choose_synonym_searcher
+from .translator import BaseTranslator, choose_translator
diff --git a/src/prod/features/attention_extractor.py b/src/prod/features/attention_extractor.py
new file mode 100644
index 0000000..1a29eaf
--- /dev/null
+++ b/src/prod/features/attention_extractor.py
@@ -0,0 +1,59 @@
+from abc import ABC, abstractmethod
+import typing as tp
+from typing import Any
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from omegaconf import DictConfig
+import numpy as np
+
+class BaseAttentionExtractor(ABC):
+ def __init__(self, model: AutoModelForSeq2SeqLM):
+ """
+ :@param model: model to extract attention from
+ """
+ super().__init__()
+ self.model = model
+
+ @abstractmethod
+ def __call__(self, tokens: tp.List[str]) -> tp.List[tp.List[float]]:
+ """
+ Extract attention scores for each token in relation to all other tokens
+
+ :@param tokens: list of tokens to extract attention for
+ :@return: list of attention scores for each token
+ """
+ pass
+
+class RandomAttentionExtractor(BaseAttentionExtractor):
+ """
+ Just for testing purposes
+ """
+ def __call__(self, tokens: tp.List[str]) -> tp.List[tp.List[float]]:
+ """
+ Extract random attention scores for each token in relation to all other tokens
+
+ :@param tokens: list of tokens to extract attention for
+ :@return: list of attention scores for each token
+ """
+ n_tokens = len(tokens)
+ unnormalized_scores = np.random.rand(n_tokens, n_tokens)
+ normalized_scores = unnormalized_scores / unnormalized_scores.sum(axis=1, keepdims=True)
+ return normalized_scores.tolist()
+
+class Seq2SeqAttentionExtractor(BaseAttentionExtractor):
+ pass
+
+
+def choose_attention_extractor(cfg: DictConfig) -> BaseAttentionExtractor:
+ """
+ Choose attention extractor based on config
+
+ :@param cfg: config to choose attention extractor from
+ :@return: attention extractor
+ """
+ if cfg.type == "random":
+ model = None
+ return RandomAttentionExtractor(model)
+ if cfg.type == "seq2seq":
+ return Seq2SeqAttentionExtractor(cfg)
+ else:
+ raise ValueError(f"Unknown attention extractor type: {cfg.attention_extractor.type}")
\ No newline at end of file
diff --git a/src/prod/features/synonym_searcher.py b/src/prod/features/synonym_searcher.py
new file mode 100644
index 0000000..742c8da
--- /dev/null
+++ b/src/prod/features/synonym_searcher.py
@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+import typing as tp
+import fasttext
+from omegaconf import DictConfig
+
+class BaseSynonymSearcher(ABC):
+ @abstractmethod
+ def __call__(self, tokens: tp.List[str], *args, return_scores: bool = False,
+ n_synonyms: int = 5, **kwargs) -> tp.List[tp.List[str]] | tp.List[tp.List[tp.Tuple[str, float]]]:
+ """
+ Search for synonyms for each token in the list
+ :@param tokens: list of tokens to find synonyms for
+ :@param return_scores: whether to return scores for synonyms
+ :@param n_synonyms: number of synonyms to return for each token
+ :@return: list of synonyms for each token or list of synonyms and scores
+ """
+ pass
+
+
+class FastTextSynonymSearcher(BaseSynonymSearcher):
+ def __init__(self, model_path: str):
+ """
+ :@param model_path: path to fasttext model
+ """
+ super().__init__()
+ self.model = fasttext.load_model(model_path)
+
+ def __call__(self, tokens: tp.List[str], *args, return_scores: bool = False,
+ n_synonyms: int = 5, **kwargs) -> tp.List[tp.List[str]] | tp.List[tp.List[tp.Tuple[str, float]]]:
+ """
+ Search for synonyms for each token in the list
+ :@param tokens: list of tokens to find synonyms for
+ :@param return_scores: whether to return scores for synonyms
+ :@param n_synonyms: number of synonyms to return for each token
+ :@return: list of synonyms for each token or list of synonyms and scores
+ """
+ synonyms = []
+ for token in tokens:
+ if return_scores:
+ synonyms.append(self.model.get_nearest_neighbors(token, k=n_synonyms))
+ else:
+ synonyms.append([synonym for _, synonym in self.model.get_nearest_neighbors(token, k=n_synonyms)])
+ return synonyms
+
+def choose_synonym_searcher(cfg: DictConfig) -> BaseSynonymSearcher:
+ """
+ Choose synonym searcher based on config
+
+ :@param cfg: config to choose synonym searcher from
+ :@return: synonym searcher
+ """
+ if cfg.type == "fasttext":
+ return FastTextSynonymSearcher(cfg.path)
+ else:
+ raise ValueError(f"Unknown synonym searcher type: {cfg.type}")
\ No newline at end of file
diff --git a/src/prod/features/translator.py b/src/prod/features/translator.py
new file mode 100644
index 0000000..f8aabf0
--- /dev/null
+++ b/src/prod/features/translator.py
@@ -0,0 +1,93 @@
+from abc import ABC, abstractmethod
+import typing as tp
+from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
+from omegaconf import DictConfig
+
+translation_dict = tp.Dict[str, str | tp.List[str]]
+
+class BaseTranslator(ABC):
+ def __init__(self, model: AutoModelForSeq2SeqLM, tokenizer: AutoTokenizer):
+ """
+ :@param model: model to translate with
+ :@param tokenizer: tokenizer to use
+ """
+ super().__init__()
+ self.model = model
+ self.tokenizer = tokenizer
+
+ @abstractmethod
+ def call_one(self, text: str, *args, **kwargs) -> translation_dict:
+ """
+ Translate one text
+
+ :@param text: text to translate
+ :@return: dict with translation, input and output tokens
+ {
+ 'translation': str,
+ 'input_tokens': list[str]
+ 'output_tokens': list[str]
+ }
+ """
+ pass
+
+ def __call__(self, texts: str | tp.List[str], *args, **kwargs) -> translation_dict | tp.List[translation_dict]:
+ """
+ Translate texts
+
+ :@param texts: text or list of texts to translate
+ :@return: dict or list of dicts with translation, input and output tokens for each text
+ {
+ 'translation': str,
+ 'input_tokens': list[str]
+ 'output_tokens': list[str]
+ }
+ """
+ if isinstance(texts, str):
+ return self.call_one(texts, *args, **kwargs)
+ return [self.call_one(text, *args, **kwargs) for text in texts]
+
+
+class Seq2SeqTranslator(BaseTranslator):
+ def __init__(self, cfg: DictConfig):
+ """
+ :@param cfg: config with model and tokenizer paths
+ """
+ name = cfg.model_and_tokenizer_name
+ model = AutoModelForSeq2SeqLM.from_pretrained(name)
+ tokenizer = AutoTokenizer.from_pretrained(name)
+ super().__init__(model, tokenizer)
+ self.gen_params = cfg.get("gen_params", {})
+
+ def get_tokens(self, text: str, is_target: bool) -> tp.List[str]:
+ """
+ Tokenize text with tokenizer
+
+ :@param text: text to tokenize
+ :@param is_target: whether text is target or source
+ :@return: list of tokens
+ """
+ if is_target:
+ with self.tokenizer.as_target_tokenizer():
+ return self.tokenizer.tokenize(text)
+ return self.tokenizer.tokenize(text)
+
+ def call_one(self, text: str, *args, **kwargs) -> translation_dict:
+ input_idx = self.tokenizer.encode(text, return_tensors="pt")
+ translation = self.model.generate(input_idx, **self.gen_params)
+ translation = self.tokenizer.batch_decode(translation, skip_special_tokens=True)[0]
+
+ input_tokens = self.get_tokens(text, is_target=False)
+ output_tokens = self.get_tokens(translation, is_target=True)
+
+ return {
+ "translation": translation,
+ "input_tokens": input_tokens,
+ "output_tokens": output_tokens
+ }
+
+
+def choose_translator(cfg: DictConfig) -> BaseTranslator:
+ if cfg.type == "seq2seq":
+ return Seq2SeqTranslator(cfg)
+ else:
+ raise ValueError(f"Unknown translator type: {cfg.type}")
\ No newline at end of file
diff --git a/src/prod/translator_app.py b/src/prod/translator_app.py
new file mode 100644
index 0000000..7971cde
--- /dev/null
+++ b/src/prod/translator_app.py
@@ -0,0 +1,70 @@
+import os
+import sys
+
+cur_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(cur_dir)
+
+from features import BaseAttentionExtractor, BaseSynonymSearcher, BaseTranslator
+from features import choose_attention_extractor, choose_synonym_searcher, choose_translator
+from omegaconf import DictConfig
+import typing as tp
+from nltk.tokenize import word_tokenize
+
+translation_type = tp.Dict[str, tp.Any]
+
+class TranslatorApp:
+ def __init__(self, cfg: DictConfig):
+ self.cfg = cfg
+ self.translator = choose_translator(cfg.translator)
+ self.src_synonym_searcher = choose_synonym_searcher(cfg.src_synonym_searcher)
+ self.dest_synonym_searcher = choose_synonym_searcher(cfg.dest_synonym_searcher)
+ self.attention_extractor = choose_attention_extractor(cfg.attention_extractor)
+
+ def _get_words(self, text: str) -> tp.List[str]:
+ """
+ Split text into words
+ :@param text: text to split
+ :@return: list of words
+ """
+ text = text.lower()
+ return word_tokenize(text)
+
+ def call_one(self, text: str) -> translation_type:
+ """
+ Return translation for one text
+ :@param text: text to translate
+ :@return: dict
+ {
+ 'translation': str,
+ 'input_tokens': list[str]
+ 'input_words': list[str]
+ 'output_tokens': list[str]
+ 'output_words': list[str]
+ 'src_synonyms': list[list[str]]
+ 'dest_synonyms': list[list[str]]
+ 'attention': list[list[float]]
+ }
+ """
+ translation = self.translator(text)
+ input_tokens = translation["input_tokens"]
+ output_tokens = translation["output_tokens"]
+ input_words = self._get_words(text)
+ output_words = self._get_words(translation["translation"])
+ src_synonyms = self.src_synonym_searcher(input_words)
+ dest_synonyms = self.dest_synonym_searcher(output_words)
+ attention = self.attention_extractor(input_tokens)
+ return {
+ "translation": translation["translation"],
+ "input_tokens": input_tokens,
+ "input_words": input_words,
+ "output_tokens": output_tokens,
+ "output_words": output_words,
+ "src_synonyms": src_synonyms,
+ "dest_synonyms": dest_synonyms,
+ "attention": attention
+ }
+
+ def __call__(self, texts: str | tp.List[str]) -> translation_type | tp.List[translation_type]:
+ if isinstance(texts, str):
+ return self.call_one(texts)
+ return [self.call_one(text) for text in texts]