DOC bring back numbering for user guide, raise build issues as errors, and fix warnings

Makefile

@@ -14,7 +14,7 @@ mypy:
 	$(CMD) mypy $(PYMODULE) $(TESTS)
 
 docs-build:
-	jb build -n -v ./doc
+	jb build -W -v ./doc
 
 test:
 	$(CMD) pytest --cov=$(PYMODULE) $(TESTS)

doc/_config.yml

@@ -39,6 +39,7 @@ sphinx:
     - 'sphinx.ext.napoleon'
     - 'sphinx.ext.viewcode'
     - 'sphinx.ext.autosummary'
+    - 'sphinx.ext.intersphinx'
   config:
     autosummary_generate: true
     add_module_names: false
@@ -49,3 +50,24 @@ sphinx:
       show-inheritance: true
       undoc-members: true
       private-members: false
+    nitpick_ignore:
+      - py:class
+      # - py:exc
+      # - py:func
+      # - py:meth
+      # - py:mod
+      - py:obj
+      # - py:var
+    intersphinx_mapping:
+      python:
+        - "https://docs.python.org/3"
+        - null
+      numpy:
+        - "https://numpy.org/doc/stable"
+        - null
+      sklearn:
+        - "https://scikit-learn.org/stable"
+        - null
+      pytorch:
+        - "https://pytorch.org/docs/stable"
+        - null

doc/_toc.yml

@@ -19,14 +19,14 @@ chapters:
   - file: code/architecture
   - file: code/user_guide
     sections:
-    - file: code/orchestrators/1_orchestrator
+    - file: code/orchestrators/0_orchestrator
       sections:
-      - file: code/orchestrators/2_prompt_sending_orchestrator
-      - file: code/orchestrators/3_red_teaming_orchestrator
-      - file: code/orchestrators/4_xpia_orchestrator
-      - file: code/orchestrators/5_scoring_orchestrator
-      - file: code/orchestrators/6_crescendo_orchestrator
-      - file: code/orchestrators/7_skeleton_key_attack
+      - file: code/orchestrators/1_prompt_sending_orchestrator
+      - file: code/orchestrators/2_red_teaming_orchestrator
+      - file: code/orchestrators/3_xpia_orchestrator
+      - file: code/orchestrators/4_scoring_orchestrator
+      - file: code/orchestrators/5_crescendo_orchestrator
+      - file: code/orchestrators/6_skeleton_key_attack
       - file: code/orchestrators/advbench_prompt_sending_orchestrator
       - file: code/orchestrators/benchmark_orchestrator
       - file: code/orchestrators/decoding_trust_stereotype_testing
@@ -36,6 +36,7 @@ chapters:
       - file: code/orchestrators/fuzzing_jailbreak_templates
       - file: code/orchestrators/harmbench_testing
       - file: code/orchestrators/hf_harmful_dataset_testing
+      - file: code/orchestrators/HITL_Scoring_Orchestrator
       - file: code/orchestrators/many_shot_jailbreak
       - file: code/orchestrators/pair_orchestrator
       - file: code/orchestrators/pku_safe_rlhf_testing
@@ -44,35 +45,35 @@ chapters:
       - file: code/orchestrators/use_huggingface_chat_target
       - file: code/orchestrators/violent_durian
       - file: code/orchestrators/xstest_bias_testing
-    - file: code/targets/1_prompt_targets
+    - file: code/targets/0_prompt_targets
       sections:
-      - file: code/targets/2_openai_chat_target
-      - file: code/targets/3_custom_targets
-      - file: code/targets/4_non_open_ai_chat_targets
-      - file: code/targets/5_non_llm_targets
-      - file: code/targets/6_multi_modal_targets
-      - file: code/targets/7_rate_limiting
-      - file: code/targets/8_http_target
+      - file: code/targets/1_openai_chat_target
+      - file: code/targets/2_custom_targets
+      - file: code/targets/3_non_open_ai_chat_targets
+      - file: code/targets/4_non_llm_targets
+      - file: code/targets/5_multi_modal_targets
+      - file: code/targets/6_rate_limiting
+      - file: code/targets/7_http_target
       - file: code/targets/open_ai_completions
       - file: code/targets/prompt_shield_target
-    - file: code/converters/1_converters
+    - file: code/converters/0_converters
       sections:
-        - file: code/converters/2_llm_converters
-        - file: code/converters/3_using_converters
-        - file: code/converters/4_audio_converters
-        - file: code/converters/5_image_converters
-        - file: code/converters/6_selectively_converting
-        - file: code/converters/7_human_converter
+        - file: code/converters/1_llm_converters
+        - file: code/converters/2_using_converters
+        - file: code/converters/3_audio_converters
+        - file: code/converters/4_image_converters
+        - file: code/converters/5_selectively_converting
+        - file: code/converters/6_human_converter
         - file: code/converters/char_swap_attack_generator
         - file: code/converters/math_prompt_converter
-    - file: code/scoring/1_scoring
+    - file: code/scoring/0_scoring
       sections:
-      - file: code/scoring/2_azure_content_safety_scorers
-      - file: code/scoring/3_true_false_scorers
-      - file: code/scoring/4_classification_scorers
-      - file: code/scoring/5_likert_scorers
-      - file: code/scoring/6_human_scorers
-      - file: code/scoring/7_refusal_scorer
+      - file: code/scoring/1_azure_content_safety_scorers
+      - file: code/scoring/2_true_false_scorers
+      - file: code/scoring/3_classification_scorers
+      - file: code/scoring/4_likert_scorers
+      - file: code/scoring/5_human_scorers
+      - file: code/scoring/6_refusal_scorer
       - file: code/scoring/prompt_shield_scorer
     - file: code/memory/0_memory
       sections:
@@ -87,9 +88,9 @@ chapters:
       - file: code/memory/9_exporting_data
       - file: code/memory/azure_embeddings
       - file: code/memory/chat_message
-    - file: code/auxiliary_attacks/1_auxiliary_attacks
+    - file: code/auxiliary_attacks/0_auxiliary_attacks
       sections:
-      - file: code/auxiliary_attacks/2_gcg_azure_ml
+      - file: code/auxiliary_attacks/1_gcg_azure_ml
     - file: deployment/README
       sections:
       - file: deployment/deploy_hf_model_aml

doc/code/architecture.md

@@ -27,13 +27,13 @@ This is the least defined component, because attacks can get *complicated*. They
 
 Orchestrators should contain all the logic for different types of attacks. For example, PAIR, tree of attack, or crescendo should be implemented primarily as orchestrators.
 
-Ways to contribute: Check out our [orchestrator docs](./orchestrators/1_orchestrator.md). There are hundreds of attacks outlined in research papers. A lot of these can be orchestrators. If you find an attack that doesn't fit the orchestrator model please tell us since we want to try to make it easier to orchestrate these. Are there scenarios you can write orchestrators for?
+Ways to contribute: Check out our [orchestrator docs](./orchestrators/0_orchestrator.md). There are hundreds of attacks outlined in research papers. A lot of these can be orchestrators. If you find an attack that doesn't fit the orchestrator model please tell us since we want to try to make it easier to orchestrate these. Are there scenarios you can write orchestrators for?
 
 ## Converters
 
 Converters are a powerful component that converts prompts to something else. They can be stacked and combined. They can be as varied as translating a text prompt into a Word document, rephrasing a prompt in 100 different ways, or adding a text overlay to an image.
 
-Ways to contribute: Check out our [converter docs](./converters/1_converters.ipynb). Are there ways prompts can be converted that would be useful for an attack?
+Ways to contribute: Check out our [converter docs](./converters/0_converters.ipynb). Are there ways prompts can be converted that would be useful for an attack?
 
 ## Target
 
@@ -43,14 +43,14 @@ This is often an LLM, but it doesn't have to be. For Cross-Domain Prompt Injecti
 
 One orchestrator can have many Prompt Targets (and in fact, converters and Scoring Engine can also use Prompt Targets to convert/score the prompt).
 
-Ways to contribute: Check out our [target docs](./targets/1_prompt_targets.md). Are there models you want to use at any stage or for different attacks?
+Ways to contribute: Check out our [target docs](./targets/0_prompt_targets.md). Are there models you want to use at any stage or for different attacks?
 
 
 ## Scoring Engine
 
 The scoring engine is a component that gives feedback to the orchestrator on what happened with the prompt. This could be as simple as "Was this prompt blocked?" or "Was our objective achieved?"
 
-Ways to contribute: Check out our [scoring docs](./scoring/1_scoring.md). Is there data you want to use to make decisions or analyze?
+Ways to contribute: Check out our [scoring docs](./scoring/0_scoring.md). Is there data you want to use to make decisions or analyze?
 
 ## Memory
 

doc/code/auxiliary_attacks/1_auxiliary_attacks.py → doc/code/auxiliary_attacks/0_auxiliary_attacks.py

@@ -25,7 +25,7 @@
 # ## GCG Suffixes
 
 # %% [markdown]
-# The [GCG demo notebook](2_gcg_azure_ml.ipynb) shows how to create an AML environment and submit a job that generates GCG suffixes, which can be appended to a base prompt to jailbreak a language model. In the example below, we compare the response generated by Phi-3-mini with and without a GCG suffix trained on that model.
+# The [GCG demo notebook](1_gcg_azure_ml.ipynb) shows how to create an AML environment and submit a job that generates GCG suffixes, which can be appended to a base prompt to jailbreak a language model. In the example below, we compare the response generated by Phi-3-mini with and without a GCG suffix trained on that model.
 #
 # First, we send a harmful prompt to Phi-3-mini without a GCG suffix. If the environment variables `PHI3_MINI_ENDPOINT` and `PHI3_MINI_KEY` are not set in your .env file, the target will default to the model with `AZURE_ML_MANAGED_ENDPOINT` and `AZURE_ML_MANAGED_KEY`.
 

doc/code/auxiliary_attacks/2_gcg_azure_ml.ipynb → doc/code/auxiliary_attacks/1_gcg_azure_ml.ipynb

@@ -5,7 +5,7 @@
    "id": "44a2ab29",
    "metadata": {},
    "source": [
-    "# Generating GCG Suffixes Using Azure Machine Learning"
+    "# 1. Generating GCG Suffixes Using Azure Machine Learning"
    ]
   },
   {

doc/code/auxiliary_attacks/2_gcg_azure_ml.py → doc/code/auxiliary_attacks/1_gcg_azure_ml.py

@@ -14,7 +14,7 @@
 # ---
 
 # %% [markdown]
-# # Generating GCG Suffixes Using Azure Machine Learning
+# # 1. Generating GCG Suffixes Using Azure Machine Learning
 
 # %% [markdown]
 # This notebook shows how to generate GCG suffixes using Azure Machine Learning (AML), which consists of three main steps:

doc/code/converters/2_llm_converters.ipynb → doc/code/converters/1_llm_converters.ipynb

@@ -5,7 +5,7 @@
    "id": "e4c3a85b",
    "metadata": {},
    "source": [
-    "# Converters with LLMs\n",
+    "# 1. Converters with LLMs\n",
     "\n",
     "Some converters use external infrastructure like attacker LLMs. `VariationConverter` is a converter that does this. However, converters like this are significantly slower to run than some simple converters, so if there is a static way to do a task, that is generally preffered."
    ]

doc/code/converters/2_llm_converters.py → doc/code/converters/1_llm_converters.py

@@ -13,7 +13,7 @@
 # ---
 
 # %% [markdown]
-# # Converters with LLMs
+# # 1. Converters with LLMs
 #
 # Some converters use external infrastructure like attacker LLMs. `VariationConverter` is a converter that does this. However, converters like this are significantly slower to run than some simple converters, so if there is a static way to do a task, that is generally preffered.
 

doc/code/converters/3_using_converters.ipynb → doc/code/converters/2_using_converters.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Using Prompt Converters\n",
+    "# 2. Using Prompt Converters\n",
     "\n",
     "Although converters can be used on their own, they should be thought of as a piece in the pipeine. Typically any orchestrator will have arguments so that prompts can be converted before being sent to the target. They can be stacked, use LLMs, and are a powerful tool.\n",
     "\n",

doc/code/converters/3_using_converters.py → doc/code/converters/2_using_converters.py

@@ -13,7 +13,7 @@
 # ---
 
 # %% [markdown]
-# # Using Prompt Converters
+# # 2. Using Prompt Converters
 #
 # Although converters can be used on their own, they should be thought of as a piece in the pipeine. Typically any orchestrator will have arguments so that prompts can be converted before being sent to the target. They can be stacked, use LLMs, and are a powerful tool.
 #

doc/code/converters/4_audio_converters.ipynb → doc/code/converters/3_audio_converters.ipynb

@@ -5,7 +5,7 @@
    "id": "8dad1cc1",
    "metadata": {},
    "source": [
-    "# Audio Converters\n",
+    "# 3. Audio Converters\n",
     "\n",
     "Converters can also be multi-modal. Because it's an abstract function used interchangeably on a single `PromptRequestPiece`, it can only deal with one input value and type per time, and have one output value and type per time. Below is an example of using `AzureSpeechTextToAudioConverter`, which has an input type of `text` and an output type of `audio_path`."
    ]

doc/code/converters/4_audio_converters.py → doc/code/converters/3_audio_converters.py

@@ -1,5 +1,5 @@
 # %% [markdown]
-# # Audio Converters
+# # 3. Audio Converters
 #
 # Converters can also be multi-modal. Because it's an abstract function used interchangeably on a single `PromptRequestPiece`, it can only deal with one input value and type per time, and have one output value and type per time. Below is an example of using `AzureSpeechTextToAudioConverter`, which has an input type of `text` and an output type of `audio_path`.
 

doc/code/converters/5_image_converters.ipynb → doc/code/converters/4_image_converters.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Image Converters\n",
+    "# 4. Image Converters\n",
     "\n",
     "Text can be added to images by using the `AddTextImageConverter`.\n",
     "The converted image file will be saved in the db/results/images folder. The `text_to_add` is used for the text to add to the image, and the `prompt` contains the image file name."

doc/code/converters/5_image_converters.py → doc/code/converters/4_image_converters.py

@@ -13,7 +13,7 @@
 # ---
 
 # %% [markdown]
-# # Image Converters
+# # 4. Image Converters
 #
 # Text can be added to images by using the `AddTextImageConverter`.
 # The converted image file will be saved in the db/results/images folder. The `text_to_add` is used for the text to add to the image, and the `prompt` contains the image file name.

doc/code/converters/6_selectively_converting.ipynb → doc/code/converters/5_selectively_converting.ipynb

@@ -4,7 +4,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Selectively Converting\n",
+    "# 5. Selectively Converting\n",
     "\n",
     "You can selectively convert strings from text converters using most orchestrators or the `convert_tokens_async` function. This function uses a `start_token` and `end_token` to determine where to do the converting (by default these are the unicode characters ⟪ and ⟫). Here is an example that uses `PromptSendingOrchestrator` to convert pieces of the text to base64."
    ]

doc/code/converters/6_selectively_converting.py → doc/code/converters/5_selectively_converting.py

@@ -13,7 +13,7 @@
 # ---
 
 # %% [markdown]
-# # Selectively Converting
+# # 5. Selectively Converting
 #
 # You can selectively convert strings from text converters using most orchestrators or the `convert_tokens_async` function. This function uses a `start_token` and `end_token` to determine where to do the converting (by default these are the unicode characters ⟪ and ⟫). Here is an example that uses `PromptSendingOrchestrator` to convert pieces of the text to base64.
 

doc/code/converters/7_human_converter.ipynb → doc/code/converters/6_human_converter.ipynb

@@ -5,7 +5,7 @@
    "id": "72a3b97b",
    "metadata": {},
    "source": [
-    "# Human in the Loop Converter\n",
+    "# 6. Human in the Loop Converter\n",
     "\n",
     "The Human in the Loop Converter allows a user to review each prompt before sending it to a target, allowing for closer moderation of multi-turn conversations.\n",
     "The user can choose to send the prompt as is, modify the prompt, or run the prompt through one of the passed-in converters before sending it.\n",

doc/code/converters/7_human_converter.py → doc/code/converters/6_human_converter.py

@@ -13,7 +13,7 @@
 # ---
 
 # %% [markdown]
-# # Human in the Loop Converter
+# # 6. Human in the Loop Converter
 #
 # The Human in the Loop Converter allows a user to review each prompt before sending it to a target, allowing for closer moderation of multi-turn conversations.
 # The user can choose to send the prompt as is, modify the prompt, or run the prompt through one of the passed-in converters before sending it.

doc/code/converters/char_swap_attack_generator.ipynb

@@ -5,7 +5,7 @@
    "id": "39195155",
    "metadata": {},
    "source": [
-    "# Generating Perturbed Prompts Using the CharSwapGenerator\n",
+    "# Generating Perturbed Prompts Using the CharSwapGenerator - optional\n",
     "\n",
     "In this script, we demonstrate how to use the `CharSwapGenerator` to generate perturbed prompts by swapping characters in words.\n",
     "The converter interacts with the Azure OpenAI API, sending prompts asynchronously through the `PromptSendingOrchestrator`.\n",

doc/code/converters/char_swap_attack_generator.py

@@ -1,5 +1,5 @@
 # %% [markdown]
-# # Generating Perturbed Prompts Using the CharSwapGenerator
+# # Generating Perturbed Prompts Using the CharSwapGenerator - optional
 #
 # In this script, we demonstrate how to use the `CharSwapGenerator` to generate perturbed prompts by swapping characters in words.
 # The converter interacts with the Azure OpenAI API, sending prompts asynchronously through the `PromptSendingOrchestrator`.

doc/code/converters/math_prompt_converter.ipynb

@@ -5,7 +5,7 @@
    "id": "985f8d5f",
    "metadata": {},
    "source": [
-    "# Jailbreaking Large Language Models with Symbolic Mathematics Using the MathPromptConverter\n",
+    "# Jailbreaking Large Language Models with Symbolic Mathematics Using the MathPromptConverter - optional\n",
     "\n",
     "This script demonstrates how to use the `MathPromptConverter` class to transform user queries into symbolic mathematical problems by applying set theory, abstract algebra, and symbolic logic.\n",
     "The converter integrates with the `OpenAIChatTarget`, and it utilizes a predefined template (`math_prompt_converter.yaml`) to dynamically handle and convert user inputs.\n",

doc/code/converters/math_prompt_converter.py

@@ -1,5 +1,5 @@
 # %% [markdown]
-# # Jailbreaking Large Language Models with Symbolic Mathematics Using the MathPromptConverter
+# # Jailbreaking Large Language Models with Symbolic Mathematics Using the MathPromptConverter - optional
 #
 # This script demonstrates how to use the `MathPromptConverter` class to transform user queries into symbolic mathematical problems by applying set theory, abstract algebra, and symbolic logic.
 # The converter integrates with the `OpenAIChatTarget`, and it utilizes a predefined template (`math_prompt_converter.yaml`) to dynamically handle and convert user inputs.

doc/code/memory/1_duck_db_memory.ipynb

@@ -5,7 +5,7 @@
    "id": "ec85323a",
    "metadata": {},
    "source": [
-    "# DuckDB Memory\n",
+    "# 1. DuckDB Memory\n",
     "\n",
     "The memory DuckDB database can be thought of as a normalized source of truth. The memory module is the primary way pyrit keeps track of requests and responses to targets and scores. Most of this is done automatically. All Prompt Targets write to memory for later retrieval. All scorers also write to memory when scoring.\n",
     "\n",

doc/code/memory/1_duck_db_memory.py

@@ -14,7 +14,7 @@
 # ---
 
 # %% [markdown]
-# # DuckDB Memory
+# # 1. DuckDB Memory
 #
 # The memory DuckDB database can be thought of as a normalized source of truth. The memory module is the primary way pyrit keeps track of requests and responses to targets and scores. Most of this is done automatically. All Prompt Targets write to memory for later retrieval. All scorers also write to memory when scoring.
 #

doc/code/memory/2_basic_memory_programming.ipynb

@@ -5,7 +5,7 @@
    "id": "00ae6a15",
    "metadata": {},
    "source": [
-    "# Basic Memory Programming Usage\n",
+    "# 2. Basic Memory Programming Usage\n",
     "\n",
     "The `pyrit.memory` module provides functionality to keep track of the conversation history, scoring, data, and more. You can use memory to read and write data. Here is an example that retrieves a normalized conversation:"
    ]

doc/code/memory/2_basic_memory_programming.py

@@ -14,7 +14,7 @@
 # ---
 
 # %% [markdown]
-# # Basic Memory Programming Usage
+# # 2. Basic Memory Programming Usage
 #
 # The `pyrit.memory` module provides functionality to keep track of the conversation history, scoring, data, and more. You can use memory to read and write data. Here is an example that retrieves a normalized conversation:
 

doc/code/memory/3_prompt_request.md

@@ -1,4 +1,4 @@
-# PromptRequestPiece and PromptRequestResposne
+# 3. PromptRequestPiece and PromptRequestResposne
 
 One of the most basic data structures is [PromptRequestPiece](../../../pyrit/models/prompt_request_piece.py) and [PromptRequestResponse](../../../pyrit/models/prompt_request_response.py).
 

doc/code/memory/4_manually_working_with_memory.md

@@ -1,11 +1,9 @@
-# Updating Memory Manually
-
+# 4. Updating Memory Manually
 
 After or during an operation or a test, it can be important to use the memory in the database in a straightforward way.
 
 There are many ways to do this, but this section gives some general ideas on how users can solve common problems. Most of this relies on using https://duckdb.org/docs/guides/sql_editors/dbeaver.html
 
-
 ## Sharing Data Between Users
 
 Eventually, we have plans to extend the `MemoryInterface` implementation to other instances. For example, it would not be a huge task to extend it to Azure SQL Server, and operators could use that as a shared database.