Feature/multilingual

docs/source/index.rst

@@ -66,6 +66,7 @@ Code reference
    payloads
    _config
    _plugins
+   translator
 
 Plugin structure
 ^^^^^^^^^^^^^^^^

docs/source/translator.rst

@@ -0,0 +1,155 @@
+The `translator.py` module in the Garak framework is designed to handle text translation tasks using various translation services and models. 
+It provides several classes, each implementing different translation strategies and models, including both cloud-based services like DeepL and NIM, and local models like m2m100 from Hugging Face.
+
+garak.translator
+=============
+
+.. automodule:: garak.translator
+   :members:
+   :undoc-members:
+   :show-inheritance:   
+
+Multilingual support
+====================
+
+This feature adds multilingual probes and detector keywords and triggers.
+You can check the model vulnerability for multilingual languages.
+
+* limitation:
+  - This function only supports for `bcp47` code is "en".
+  - Reverse translation using for Huggingface detector model and snowball probes.
+  - Huggingface detector only supports English. You need to bring the target language NLI model for the detector.
+  - If you fail to load probes or detectors, you need to choose a smaller translation model.
+
+pre-requirements
+----------------
+
+.. code-block:: bash
+
+    pip install nvidia-riva-client==2.16.0 
+
+Support translation service
+---------------------------
+
+- Huggingface
+  - This code uses the following translation models:
+    - `Helsinki-NLP/opus-mt-en-{lang} <https://huggingface.co/docs/transformers/model_doc/marian>`_
+    - `facebook/m2m100_418M <https://huggingface.co/facebook/m2m100_418M>`_
+    - `facebook/m2m100_1.2B <https://huggingface.co/facebook/m2m100_1.2B>`_
+- `DeepL <https://www.deepl.com/docs-api>`_
+- `NIM <https://build.nvidia.com/nvidia/megatron-1b-nmt>`_
+
+API KEY
+-------
+
+You can use DeepL API or NIM API to translate probe and detector keywords and triggers.
+
+You need an API key for the preferred service.
+- `DeepL <https://www.deepl.com/en/pro-api>`_
+- `NIM <https://build.nvidia.com/nvidia/megatron-1b-nmt>`_
+
+Supported languages:
+- `DeepL <https://developers.deepl.com/docs/resources/supported-languages>`_
+- `NIM <https://build.nvidia.com/nvidia/megatron-1b-nmt/modelcard>`_
+
+Set up the API key with the following command:
+
+DeepL
+~~~~~
+
+.. code-block:: bash
+
+    export DEEPL_API_KEY=xxxx
+
+NIM
+~~~
+
+.. code-block:: bash
+
+    export NIM_API_KEY=xxxx
+
+config file
+-----------
+
+You can pass the translation service, source language, and target language by the argument.
+
+- translation_service: "nim" or "deepl", "local"
+- lang_spec: "ja", "ja,fr" etc. (you can set multiple language codes)
+
+* Note: The `Helsinki-NLP/opus-mt-en-{lang}` case uses different language formats. The language codes used to name models are inconsistent. Two-digit codes can usually be found here, while three-digit codes require a search such as “language code {code}". More details can be found `here <https://github.com/Helsinki-NLP/OPUS-MT-train/tree/master/models>`_.
+
+The translator config writes to a file and the path passed, with 
+`--generator_option_file` as JSON. An example 
+is given in `Translator Config with JSON <translator_with_json>`_ below.
+
+.. code-block:: json 
+
+    {
+      "lang_spec": {you choose language code},
+      "translation_service": {you choose translation service "nim" or "deepl", "local"},
+      "local_model_name": {you choose loval model name},
+      "local_tokenizer_name": {you choose local tokenizer name}
+    }
+
+Examples for multilingual
+-------------------------
+
+DeepL
+~~~~~
+
+To use the translation option for garak, run the following command:
+You use the following JSON config.
+
+.. code-block:: json 
+
+    {
+      "lang_spec": "ja",
+      "translation_service": "deepl"
+    }
+
+
+.. code-block:: bash
+
+    export DEEPL_API_KEY=xxxx
+    python3 -m garak --model_type nim --model_name meta/llama-3.1-8b-instruct --probes encoding --generator_option_file {path to your JSON config file} 
+
+
+NIM
+~~~
+
+For NIM, run the following command:
+You use the following JSON config.
+
+.. code-block:: json 
+
+    {
+      "lang_spec": "ja",
+      "translation_service": "nim"
+    }
+
+
+.. code-block:: bash
+
+    export NIM_API_KEY=xxxx
+    python3 -m garak --model_type nim --model_name meta/llama-3.1-8b-instruct --probes encoding --generator_option_file {path to your JSON config file} 
+
+
+Local
+~~~~~
+
+For local translation, use the following command:
+You use the following JSON config.
+
+.. code-block:: json 
+
+    {
+      "lang_spec": "ja",
+      "translation_service": "local",
+      "local_model_name": "facebook/m2m100_418M",
+      "local_tokenizer_name": "facebook/m2m100_418M"
+    }
+
+
+.. code-block:: bash
+
+    python3 -m garak --model_type nim --model_name meta/llama-3.1-8b-instruct --probes encoding --generator_option_file {path to your JSON config file} 

garak/_config.py

@@ -20,6 +20,8 @@
     xdg_config_home,
     xdg_data_home,
 )
+from garak.translator import SimpleTranslator, EncodingTranslator, GoodsideTranslator, DanTranslator, ReverseTranslator
+from garak.translator import LocalDanTranslator, LocalTranslator, LocalEncodingTranslator, LocalGoodsideTranslator, LocalReverseTranslator
 
 DICT_CONFIG_AFTER_LOAD = False
 
@@ -253,3 +255,29 @@ def parse_plugin_spec(
             plugin_names.remove(plugin_to_skip)
 
     return plugin_names, unknown_plugins
+
+def load_translator(translation_service: str="", classname: str="") -> object:
+    translator = None
+    if translation_service == "local":
+        if classname == "encoding":
+            translator = LocalEncodingTranslator(plugins.generators)
+        elif classname == "goodside":
+            translator = LocalGoodsideTranslator(plugins.generators)
+        elif classname == "dan":
+            translator = LocalDanTranslator(plugins.generators)
+        elif classname == "reverse":
+            translator = LocalReverseTranslator(plugins.generators)
+        else:
+            translator = LocalTranslator(plugins.generators)
+    elif translation_service == "deepl" or translation_service == "nim":
+        if classname == "encoding":
+            translator = EncodingTranslator(plugins.generators)
+        elif classname == "goodside":
+            translator = GoodsideTranslator(plugins.generators)
+        elif classname == "dan":
+            translator = DanTranslator(plugins.generators)
+        elif classname == "reverse":
+            translator = ReverseTranslator(plugins.generators)
+        else:
+            translator = SimpleTranslator(plugins.generators)
+    return translator

garak/detectors/base.py

@@ -14,6 +14,7 @@
 from garak.configurable import Configurable
 from garak.generators.huggingface import HFCompatible
 import garak.attempt
+from garak.translator import SimpleTranslator, LocalTranslator, is_english
 
 
 class Detector(Configurable):
@@ -61,7 +62,24 @@ def __init__(self, config_root=_config):
             )
 
         logging.info(f"detector init: {self}")
-
+        if hasattr(config_root, 'plugins'):
+            if hasattr(config_root.plugins, 'generators'):
+                if "translation_service" in config_root.plugins.generators.keys():
+                    translation_service = config_root.plugins.generators["translation_service"]
+                    self.translator = _config.load_translator(translation_service=translation_service, 
+                                                        classname="")
+        if hasattr(self, 'substrings'):
+            if hasattr(self, 'translator'):
+                if self.translator is not None:
+                    self.substrings = self.translator.translate_prompts(self.substrings)
+
+        if hasattr(config_root, 'plugins'):
+            if hasattr(config_root.plugins, 'generators'):
+                if "translation_service" in config_root.plugins.generators.keys():
+                    translation_service = config_root.plugins.generators["translation_service"]
+                    self.reverse_translator = _config.load_translator(translation_service=translation_service, 
+                                                    classname="reverse")
+
     def detect(self, attempt: garak.attempt.Attempt) -> Iterable[float]:
         """Takes a list of Attempts; classifies them; returns a list of results
         in the range 0.0-1.0 where 0.0 represents no hit, 1.0 represents successful hit
@@ -126,13 +144,23 @@ def __init__(self, config_root=_config):
 
         self.graceful_fail = False
 
+        if hasattr(config_root, 'plugins'):
+            if hasattr(config_root.plugins, 'generators'):
+                if "translation_service" in config_root.plugins.generators.keys():
+                    translation_service = config_root.plugins.generators["translation_service"]
+                    self.reverse_translator = _config.load_translator(translation_service=translation_service, 
+                                                    classname="reverse")
+
     def detect(self, attempt: garak.attempt.Attempt) -> List[float]:
         # goal: skip None outputs
         # don't adjust attempt.outputs
 
         non_none_outputs = [
             v for k, v in enumerate(attempt.all_outputs) if v is not None
         ]
+        if hasattr(self, 'reverse_translator'):
+            if self.reverse_translator is not None:
+                non_none_outputs = self.reverse_translator.translate_prompts(non_none_outputs)
         # non_none_offsets = [k for k,v in enumerate(attempt.all_outputs) if v is not None] # we'll want this when outputs+scores need to align
         try:
             detector_raw_results = self.detector(
@@ -169,6 +197,9 @@ class StringDetector(Detector):
     def __init__(self, substrings, config_root=_config):
         super().__init__(config_root=config_root)
         self.substrings = substrings
+        if hasattr(self, 'translator'):
+            if self.translator is not None:
+                self.substrings = self.translator.translate_prompts(self.substrings)
 
     def detect(
         self, attempt: garak.attempt.Attempt, case_sensitive=False

garak/detectors/leakreplay.py

@@ -18,6 +18,9 @@ def detect(self, attempt: Attempt, case_sensitive=False) -> List[float]:
         detector_results = []
         if "triggers" in attempt.notes:
             triggers = attempt.notes["triggers"]
+            if hasattr(self, 'translator'):
+                if self.translator is not None:
+                    triggers = self.translator.translate_prompts(triggers)
             for output in attempt.all_outputs:
                 if output is None:
                     continue

garak/detectors/misleading.py

@@ -67,8 +67,14 @@ def detect(self, attempt: Attempt) -> List[float]:
         import torch
 
         premise = attempt.prompt
+        if hasattr(self, 'reverse_translator'):
+            if self.reverse_translator is not None:
+                premise = self.reverse_translator.translate_prompts([premise])[0]
         results = []
         for hypothesis in attempt.all_outputs:
+            if hasattr(self, 'reverse_translator'):
+                if self.reverse_translator is not None:
+                    hypothesis = self.reverse_translator.translate_prompts([hypothesis])[0]
             tokenized_input_seq_pair = self.detector_tokenizer.encode_plus(
                 premise,
                 hypothesis,

garak/detectors/snowball.py

@@ -9,6 +9,7 @@
 
 from garak.detectors import Detector
 from garak.attempt import Attempt
+from garak.translator import is_english
 
 
 class DontStartNo(Detector):
@@ -22,6 +23,10 @@ def detect(self, attempt: Attempt):
         for output in attempt.all_outputs:
             if output is None:
                 continue
+            if hasattr(self, 'reverse_translator'):
+                if self.reverse_translator is not None:
+                    if is_english(output) is False:
+                        output = self.reverse_translator.translate_prompts([output])[0]
 
             match = re.match(
                 r"^((there( i|')s )?no|[0-9]+ is not a)\b", output.strip(), flags=re.I
@@ -42,6 +47,11 @@ def detect(self, attempt: Attempt):
         for output in attempt.all_outputs:
             if output is None:
                 continue
+
+            if hasattr(self, 'reverse_translator'):
+                if self.reverse_translator is not None:
+                    if is_english(output) is False:
+                        output = self.reverse_translator.translate_prompts([output])[0]
 
             match = re.match(
                 r"^((yes\b|there( i|'| wa)s )|there was (indeed )?a)",