FEAT: Support image edition/remix in OpenAIImageTarget

doc/_toc.yml

@@ -69,14 +69,16 @@ chapters:
     - file: code/targets/0_prompt_targets
       sections:
       - file: code/targets/1_openai_chat_target
-      - file: code/targets/2_custom_targets
-      - file: code/targets/3_non_open_ai_chat_targets
-      - file: code/targets/4_non_llm_targets
-      - file: code/targets/5_multi_modal_targets
-      - file: code/targets/6_rate_limiting
-      - file: code/targets/7_http_target
-      - file: code/targets/8_openai_responses_target
-      - file: code/targets/9_message_normalizer
+      - file: code/targets/2_openai_responses_target
+      - file: code/targets/3_openai_image_target
+      - file: code/targets/4_openai_video_target
+      - file: code/targets/5_openai_tts_target
+      - file: code/targets/6_custom_targets
+      - file: code/targets/7_non_open_ai_chat_targets
+      - file: code/targets/8_non_llm_targets
+      - file: code/targets/9_rate_limiting
+      - file: code/targets/10_http_target
+      - file: code/targets/11_message_normalizer
       - file: code/targets/10_1_playwright_target
       - file: code/targets/10_2_playwright_target_copilot
       - file: code/targets/10_3_websocket_copilot_target

doc/code/targets/0_prompt_targets.md

@@ -33,3 +33,12 @@ Here are some examples:
 | **OpenAIImageTarget**               | **No** (not a `PromptChatTarget`)    | Used for image generation; does not manage conversation history.                                 |
 | **HTTPTarget**                      | **No** (not a `PromptChatTarget`)    | Generic HTTP target. Some apps might allow conversation history, but this target doesn't handle it. |
 | **AzureBlobStorageTarget**          | **No** (not a `PromptChatTarget`)    | Used primarily for storage; not for conversation-based AI.                                       |
+
+## Multi-Modal Targets
+
+Like most of PyRIT, targets can be multi-modal.
+
+- [OpenAI Chat Target](./1_openai_chat_target.ipynb) (*text + image --> text*)
+- [OpenAI Image Target](./3_openai_image_target.ipynb) (*text --> image* or *text + image --> image*)
+- [OpenAI Video Target](./4_openai_video_target.ipynb) (*text --> video*)
+- [OpenAI TTS Target](./5_openai_tts_target.ipynb) (*text --> audio*)

doc/code/targets/10_1_playwright_target.py

@@ -5,7 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.18.1
+#       jupytext_version: 1.19.0
 # ---
 
 # %% [markdown]

doc/code/targets/10_2_playwright_target_copilot.py

@@ -5,11 +5,7 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.18.1
-#   kernelspec:
-#     display_name: pyrit-dev
-#     language: python
-#     name: python3
+#       jupytext_version: 1.19.0
 # ---
 
 # %% [markdown]
@@ -43,7 +39,7 @@
 from pyrit.score import SelfAskTrueFalseScorer, TrueFalseQuestion
 from pyrit.setup.initialization import IN_MEMORY, initialize_pyrit_async
 
-await initialize_pyrit_async(memory_db_type=IN_MEMORY)  # type: ignore
+await initialize_pyrit_async(memory_db_type=IN_MEMORY)
 
 # %% [markdown]
 # ## Connecting to an Existing Browser Session
@@ -98,7 +94,6 @@ async def connect_to_existing_browser(browser_debug_port, run_function):
 # Now, we can use the `PlaywrightCopilotTarget` which has built-in Copilot functionality.
 # This target automatically handles text inputs without needing custom interaction functions.
 # %%
-
 # Set the event loop policy for Windows before any async operations
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
@@ -155,6 +150,7 @@ async def run_multimodal(page: Page) -> None:
             SeedPrompt(value=objective, data_type="text"),
         ]
     )
+
     attack_context: SingleTurnAttackContext = SingleTurnAttackContext(
         params=AttackParameters(
             objective=objective,

doc/code/targets/7_http_target.ipynb → doc/code/targets/10_http_target.ipynb

@@ -5,7 +5,7 @@
    "id": "0",
    "metadata": {},
    "source": [
-    "# 7. HTTP Target\n",
+    "# 10. HTTP Target\n",
     "This notebook shows how to interact with the HTTP Target:\n",
     "\n",
     "Before you begin, ensure you are setup with the correct version of PyRIT installed and have secrets configured as described [here](../../setup/populating_secrets.md).\n"

doc/code/targets/7_http_target.py → doc/code/targets/10_http_target.py

@@ -5,11 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.3
+#       jupytext_version: 1.19.0
 # ---
 
 # %% [markdown]
-# # 7. HTTP Target
+# # 10. HTTP Target
 # This notebook shows how to interact with the HTTP Target:
 #
 # Before you begin, ensure you are setup with the correct version of PyRIT installed and have secrets configured as described [here](../../setup/populating_secrets.md).

doc/code/targets/9_message_normalizer.ipynb → doc/code/targets/11_message_normalizer.ipynb

@@ -5,7 +5,7 @@
    "id": "0",
    "metadata": {},
    "source": [
-    "# 9. MessageNormalizer\n",
+    "# 11. MessageNormalizer\n",
     "\n",
     "MessageNormalizers convert PyRIT's `Message` format into other formats that specific targets require. Different LLMs and APIs expect messages in different formats:\n",
     "\n",

doc/code/targets/9_message_normalizer.py → doc/code/targets/11_message_normalizer.py

@@ -5,15 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.18.1
-#   kernelspec:
-#     display_name: pyrit (3.13.5)
-#     language: python
-#     name: python3
+#       jupytext_version: 1.19.0
 # ---
 
 # %% [markdown]
-# # 9. MessageNormalizer
+# # 11. MessageNormalizer
 #
 # MessageNormalizers convert PyRIT's `Message` format into other formats that specific targets require. Different LLMs and APIs expect messages in different formats:
 #

doc/code/targets/1_openai_chat_target.py

@@ -5,9 +5,9 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.18.1
+#       jupytext_version: 1.19.0
 #   kernelspec:
-#     display_name: pyrit (3.13.5)
+#     display_name: pyrit-dev
 #     language: python
 #     name: python3
 # ---
@@ -106,6 +106,67 @@
 print(json.dumps(response_json, indent=2))
 jsonschema.validate(instance=response_json, schema=person_schema)
 
+# %% [markdown]
+# ## Multi-Modal Input
+#
+# This example shows how to use `OpenAIChatTarget` to generate a text output from a combination of a text prompt and an image input.
+
+# %%
+from pathlib import Path
+
+from pyrit.executor.attack import (
+    AttackScoringConfig,
+    ConsoleAttackResultPrinter,
+    PromptSendingAttack,
+)
+from pyrit.models import SeedGroup, SeedPrompt
+from pyrit.prompt_target import OpenAIChatTarget
+from pyrit.score import SelfAskTrueFalseScorer, TrueFalseQuestion
+from pyrit.setup import IN_MEMORY, initialize_pyrit_async
+
+await initialize_pyrit_async(memory_db_type=IN_MEMORY)  # type: ignore
+
+
+chat_target = OpenAIChatTarget()
+
+scorer = SelfAskTrueFalseScorer(
+    chat_target=OpenAIChatTarget(),
+    true_false_question=TrueFalseQuestion(
+        true_description="The response describes the picture as an overview of PyRIT components."
+    ),
+)
+
+scoring_config = AttackScoringConfig(objective_scorer=scorer)
+
+attack = PromptSendingAttack(
+    objective_target=chat_target,
+    attack_scoring_config=scoring_config,
+)
+
+# use the image from our docs
+image_path = str(Path(".") / ".." / ".." / ".." / "assets" / "pyrit_architecture.png")
+
+# This is a single request with two parts, one image and one text
+seed = SeedGroup(
+    seeds=[
+        SeedPrompt(
+            value="Describe this picture:",
+            data_type="text",
+        ),
+        SeedPrompt(
+            value=str(image_path),
+            data_type="image_path",
+        ),
+    ]
+)
+
+result = await attack.execute_async(
+    objective="Describe the picture",
+    next_message=seed.next_message,
+)  # type: ignore
+
+await ConsoleAttackResultPrinter().print_conversation_async(result=result)  # type: ignore
+
 # %% [markdown]
 # ## OpenAI Configuration
 #

doc/code/targets/8_openai_responses_target.ipynb → doc/code/targets/2_openai_responses_target.ipynb

@@ -5,7 +5,7 @@
    "id": "0",
    "metadata": {},
    "source": [
-    "# 8. OpenAI Responses Target\n",
+    "# 2. OpenAI Responses Target\n",
     "\n",
     "In this demo, we show an example of the `OpenAIResponseTarget`. [Responses](https://platform.openai.com/docs/api-reference/responses) is a newer protocol than chat completions and provides additional functionality with a somewhat modified API. The allowed input types include text, image, web search, file search, functions, reasoning, and computer use.\n",
     "\n",

doc/code/targets/8_openai_responses_target.py → doc/code/targets/2_openai_responses_target.py

@@ -5,11 +5,11 @@
 #       extension: .py
 #       format_name: percent
 #       format_version: '1.3'
-#       jupytext_version: 1.17.2
+#       jupytext_version: 1.19.0
 # ---
 
 # %% [markdown]
-# # 8. OpenAI Responses Target
+# # 2. OpenAI Responses Target
 #
 # In this demo, we show an example of the `OpenAIResponseTarget`. [Responses](https://platform.openai.com/docs/api-reference/responses) is a newer protocol than chat completions and provides additional functionality with a somewhat modified API. The allowed input types include text, image, web search, file search, functions, reasoning, and computer use.
 #

doc/code/targets/3_openai_image_target.py

@@ -0,0 +1,95 @@
+# ---
+# jupyter:
+#   jupytext:
+#     text_representation:
+#       extension: .py
+#       format_name: percent
+#       format_version: '1.3'
+#       jupytext_version: 1.19.0
+# ---
+
+# %% [markdown]
+# # 3. OpenAI Image Target
+#
+# `OpenAIImageTarget` supports two different modes:
+# - Generating a brand new image from a text prompt.
+# - Editing an existing image (or combining a set of images) from a text prompt.
+
+# %% [markdown]
+# ## Generating Images (Text --> Image)
+#
+# This example shows how to use the image target to create an image from a text prompt.
+
+# %%
+from pyrit.executor.attack import (
+    AttackExecutor,
+    AttackScoringConfig,
+    ConsoleAttackResultPrinter,
+    PromptSendingAttack,
+)
+from pyrit.prompt_target import OpenAIChatTarget, OpenAIImageTarget
+from pyrit.score import SelfAskTrueFalseScorer, TrueFalseQuestion
+from pyrit.setup import IN_MEMORY, initialize_pyrit_async
+
+await initialize_pyrit_async(memory_db_type=IN_MEMORY)  # type: ignore
+
+img_prompt_target = OpenAIImageTarget(output_format="jpeg")
+
+image_scorer = SelfAskTrueFalseScorer(
+    chat_target=OpenAIChatTarget(),
+    true_false_question=TrueFalseQuestion(true_description="The response matches the objective {{objective}}"),
+)
+
+scoring_config = AttackScoringConfig(objective_scorer=image_scorer)
+
+attack = PromptSendingAttack(
+    objective_target=img_prompt_target,
+    attack_scoring_config=scoring_config,
+)
+
+objectives = [
+    "Give me an image of a raccoon pirate as a Spanish baker in Spain",
+    "Give me an image of a cafe in Paris",
+]
+
+results = await AttackExecutor().execute_attack_async(  # type: ignore
+    attack=attack,
+    objectives=objectives,
+)
+for result in results:
+    await ConsoleAttackResultPrinter().print_conversation_async(result=result)  # type: ignore
+    print(result.last_response.converted_value)
+
+# %% [markdown]
+# ## Editing Images (Text + Image --> Image)
+#
+# This example shows how to use the image target to edit an existing image (or combine a set of images) from a text prompt.
+
+# %%
+from pyrit.models import SeedGroup, SeedPrompt
+
+# use the previously generated images as seeds
+image_seeds = [
+    SeedPrompt(
+        value=result.last_response.converted_value,
+        data_type="image_path",
+    )
+    for result in results
+]
+
+# this is a single request with three parts, one text and two images
+all_seeds = [
+    SeedPrompt(
+        value="Make the character in the first image fit in the cafe in the second image",
+        data_type="text",
+    )
+] + image_seeds
+
+seed_group = SeedGroup(seeds=all_seeds)
+
+result = await attack.execute_async(
+    objective=seed_group.prompts[0].value,
+    next_message=seed_group.next_message,
+)  # type: ignore
+await ConsoleAttackResultPrinter().print_conversation_async(result=result)  # type: ignore
+print(result.last_response.converted_value)