diff --git a/doc/api.rst b/doc/api.rst index 89738196c..8fbec2512 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -272,6 +272,7 @@ API Reference :toctree: _autosummary/ class_name_to_snake_case + ConverterIdentifier Identifiable Identifier IdentifierT diff --git a/pyrit/exceptions/exception_context.py b/pyrit/exceptions/exception_context.py index 11515a413..bce5b9238 100644 --- a/pyrit/exceptions/exception_context.py +++ b/pyrit/exceptions/exception_context.py @@ -13,7 +13,9 @@ from contextvars import ContextVar from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union + +from pyrit.identifiers import Identifier class ComponentRole(Enum): @@ -191,7 +193,7 @@ def execution_context( component_role: ComponentRole, attack_strategy_name: Optional[str] = None, attack_identifier: Optional[Dict[str, Any]] = None, - component_identifier: Optional[Dict[str, Any]] = None, + component_identifier: Optional[Union[Identifier, Dict[str, Any]]] = None, objective_target_conversation_id: Optional[str] = None, objective: Optional[str] = None, ) -> ExecutionContextManager: @@ -203,6 +205,7 @@ def execution_context( attack_strategy_name: The name of the attack strategy class. attack_identifier: The identifier from attack.get_identifier(). component_identifier: The identifier from component.get_identifier(). + Can be an Identifier object or a dict (legacy format). objective_target_conversation_id: The objective target conversation ID if available. objective: The attack objective if available. @@ -212,15 +215,22 @@ def execution_context( # Extract endpoint and component_name from component_identifier if available endpoint = None component_name = None + component_id_dict: Optional[Dict[str, Any]] = None if component_identifier: - endpoint = component_identifier.get("endpoint") - component_name = component_identifier.get("__type__") + if isinstance(component_identifier, Identifier): + endpoint = getattr(component_identifier, "endpoint", None) + component_name = component_identifier.class_name + component_id_dict = component_identifier.to_dict() + else: + endpoint = component_identifier.get("endpoint") + component_name = component_identifier.get("__type__") + component_id_dict = component_identifier context = ExecutionContext( component_role=component_role, attack_strategy_name=attack_strategy_name, attack_identifier=attack_identifier, - component_identifier=component_identifier, + component_identifier=component_id_dict, objective_target_conversation_id=objective_target_conversation_id, endpoint=endpoint, component_name=component_name, diff --git a/pyrit/executor/attack/multi_turn/chunked_request.py b/pyrit/executor/attack/multi_turn/chunked_request.py index 7749e11c0..feabb9821 100644 --- a/pyrit/executor/attack/multi_turn/chunked_request.py +++ b/pyrit/executor/attack/multi_turn/chunked_request.py @@ -362,7 +362,7 @@ async def _score_combined_value_async( component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective=objective, ): scores = await self._objective_scorer.score_text_async(text=combined_value, objective=objective) diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index a6a7d2049..7a274053c 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -636,7 +636,7 @@ async def _check_refusal_async(self, context: CrescendoAttackContext, objective: component_role=ComponentRole.REFUSAL_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._refusal_scorer.get_identifier().to_dict(), + component_identifier=self._refusal_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): @@ -666,7 +666,7 @@ async def _score_response_async(self, *, context: CrescendoAttackContext) -> Sco component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): diff --git a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py index 0b3464aaa..c451af7c9 100644 --- a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py +++ b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py @@ -373,7 +373,7 @@ async def _evaluate_response_async(self, *, response: Message, objective: str) - component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict() if self._objective_scorer else None, + component_identifier=self._objective_scorer.get_identifier() if self._objective_scorer else None, objective=objective, ): scoring_results = await Scorer.score_response_async( diff --git a/pyrit/executor/attack/multi_turn/red_teaming.py b/pyrit/executor/attack/multi_turn/red_teaming.py index 619446bcc..33b2c75d7 100644 --- a/pyrit/executor/attack/multi_turn/red_teaming.py +++ b/pyrit/executor/attack/multi_turn/red_teaming.py @@ -573,7 +573,7 @@ async def _score_response_async(self, *, context: MultiTurnAttackContext[Any]) - component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index d71875333..cfad81d3a 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -639,7 +639,7 @@ async def _score_response_async(self, *, response: Message, objective: str) -> N component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self._attack_strategy_name, attack_identifier=self._attack_id, - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=self.objective_target_conversation_id, objective=objective, ): diff --git a/pyrit/executor/attack/single_turn/prompt_sending.py b/pyrit/executor/attack/single_turn/prompt_sending.py index 299aba643..3b3f62350 100644 --- a/pyrit/executor/attack/single_turn/prompt_sending.py +++ b/pyrit/executor/attack/single_turn/prompt_sending.py @@ -355,7 +355,7 @@ async def _evaluate_response_async( component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict() if self._objective_scorer else None, + component_identifier=self._objective_scorer.get_identifier() if self._objective_scorer else None, objective=objective, ): scoring_results = await Scorer.score_response_async( diff --git a/pyrit/identifiers/__init__.py b/pyrit/identifiers/__init__.py index 8ca875ca3..a6e0d759b 100644 --- a/pyrit/identifiers/__init__.py +++ b/pyrit/identifiers/__init__.py @@ -7,6 +7,7 @@ class_name_to_snake_case, snake_case_to_class_name, ) +from pyrit.identifiers.converter_identifier import ConverterIdentifier from pyrit.identifiers.identifiable import Identifiable, IdentifierT, LegacyIdentifiable from pyrit.identifiers.identifier import ( Identifier, @@ -16,6 +17,7 @@ __all__ = [ "class_name_to_snake_case", + "ConverterIdentifier", "Identifiable", "Identifier", "IdentifierT", diff --git a/pyrit/identifiers/converter_identifier.py b/pyrit/identifiers/converter_identifier.py new file mode 100644 index 000000000..777672a93 --- /dev/null +++ b/pyrit/identifiers/converter_identifier.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, Type, cast + +from pyrit.identifiers.identifier import Identifier + + +@dataclass(frozen=True) +class ConverterIdentifier(Identifier): + """ + Identifier for PromptConverter instances. + + This frozen dataclass extends Identifier with converter-specific fields. + It provides a structured way to identify and track converters used in + prompt transformations. + """ + + supported_input_types: Tuple[str, ...] = field(kw_only=True) + """The input data types supported by this converter (e.g., ('text',), ('image', 'text')).""" + + supported_output_types: Tuple[str, ...] = field(kw_only=True) + """The output data types produced by this converter.""" + + sub_identifier: Optional[List["ConverterIdentifier"]] = None + """List of sub-converter identifiers for composite converters like ConverterPipeline.""" + + target_info: Optional[Dict[str, Any]] = None + """Information about the prompt target used by the converter (for LLM-based converters).""" + + converter_specific_params: Optional[Dict[str, Any]] = None + """Additional converter-specific parameters.""" + + @classmethod + def from_dict(cls: Type["ConverterIdentifier"], data: dict[str, Any]) -> "ConverterIdentifier": + """ + Create a ConverterIdentifier from a dictionary (e.g., retrieved from database). + + Extends the base Identifier.from_dict() to recursively reconstruct + nested ConverterIdentifier objects in sub_identifier. + + Args: + data: The dictionary representation. + + Returns: + ConverterIdentifier: A new ConverterIdentifier instance. + """ + # Create a mutable copy + data = dict(data) + + # Recursively reconstruct sub_identifier if present + if "sub_identifier" in data and data["sub_identifier"] is not None: + data["sub_identifier"] = [ + ConverterIdentifier.from_dict(sub) if isinstance(sub, dict) else sub for sub in data["sub_identifier"] + ] + + # Convert supported_input_types and supported_output_types from list to tuple if needed + if "supported_input_types" in data and data["supported_input_types"] is not None: + if isinstance(data["supported_input_types"], list): + data["supported_input_types"] = tuple(data["supported_input_types"]) + else: + # Provide default for legacy dicts that don't have this field + data["supported_input_types"] = () + + if "supported_output_types" in data and data["supported_output_types"] is not None: + if isinstance(data["supported_output_types"], list): + data["supported_output_types"] = tuple(data["supported_output_types"]) + else: + # Provide default for legacy dicts that don't have this field + data["supported_output_types"] = () + + # Delegate to parent class for standard processing + result = Identifier.from_dict.__func__(cls, data) # type: ignore[attr-defined] + return cast(ConverterIdentifier, result) diff --git a/pyrit/identifiers/identifiable.py b/pyrit/identifiers/identifiable.py index b588918d8..94108e6ea 100644 --- a/pyrit/identifiers/identifiable.py +++ b/pyrit/identifiers/identifiable.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Generic, TypeVar +from typing import Generic, Optional, TypeVar from pyrit.identifiers.identifier import Identifier @@ -37,29 +37,32 @@ class Identifiable(ABC, Generic[IdentifierT]): Generic over IdentifierT, allowing subclasses to specify their exact identifier type for strong typing support. - Subclasses must: - 1. Implement `_build_identifier()` to construct their specific identifier - 2. Implement `get_identifier()` to return the typed identifier (can use lazy building) + Subclasses must implement `_build_identifier()` to construct their specific identifier. + The `get_identifier()` method is provided and uses lazy building with caching. """ + _identifier: Optional[IdentifierT] = None + @abstractmethod - def _build_identifier(self) -> None: + def _build_identifier(self) -> IdentifierT: """ - Build the identifier for this object. + Build and return the identifier for this object. - Subclasses must implement this method to construct their specific identifier type - and store it in an instance variable (typically `_identifier`). + Subclasses must implement this method to construct their specific identifier type. + This method is called lazily on first access via `get_identifier()`. - This method is typically called lazily on first access via `get_identifier()`. + Returns: + IdentifierT: The constructed identifier for this component. """ raise NotImplementedError("Subclasses must implement _build_identifier") - @abstractmethod def get_identifier(self) -> IdentifierT: """ - Get the typed identifier for this object. + Get the typed identifier for this object. Built lazily on first access. Returns: IdentifierT: The identifier for this component. """ - ... + if self._identifier is None: + self._identifier = self._build_identifier() + return self._identifier diff --git a/pyrit/identifiers/identifier.py b/pyrit/identifiers/identifier.py index 8e5265d4b..3d698e717 100644 --- a/pyrit/identifiers/identifier.py +++ b/pyrit/identifiers/identifier.py @@ -64,7 +64,7 @@ def _compute_hash(self) -> str: """ Compute a stable SHA256 hash from storable identifier fields. - Fields marked with metadata={"exclude_from_storage": True}, 'hash', and 'name' + Fields marked with metadata={"exclude_from_storage": True}, 'hash', and 'unique_name' are excluded from the hash computation. Returns: diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 1401c2e24..dd787f718 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -31,7 +31,7 @@ from sqlalchemy.types import Uuid from pyrit.common.utils import to_sha256 -from pyrit.identifiers import ScorerIdentifier +from pyrit.identifiers import ConverterIdentifier, ScorerIdentifier from pyrit.models import ( AttackOutcome, AttackResult, @@ -164,7 +164,7 @@ class PromptMemoryEntry(Base): labels: Mapped[dict[str, str]] = mapped_column(JSON) prompt_metadata: Mapped[dict[str, Union[str, int]]] = mapped_column(JSON) targeted_harm_categories: Mapped[Optional[List[str]]] = mapped_column(JSON) - converter_identifiers: Mapped[Optional[List[dict[str, str]]]] = mapped_column(JSON) + converter_identifiers: Mapped[Optional[List[Dict[str, str]]]] = mapped_column(JSON) prompt_target_identifier: Mapped[dict[str, str]] = mapped_column(JSON) attack_identifier: Mapped[dict[str, str]] = mapped_column(JSON) response_error: Mapped[Literal["blocked", "none", "processing", "unknown"]] = mapped_column(String, nullable=True) @@ -207,7 +207,7 @@ def __init__(self, *, entry: MessagePiece): self.labels = entry.labels self.prompt_metadata = entry.prompt_metadata self.targeted_harm_categories = entry.targeted_harm_categories - self.converter_identifiers = entry.converter_identifiers + self.converter_identifiers = [conv.to_dict() for conv in entry.converter_identifiers] self.prompt_target_identifier = entry.prompt_target_identifier self.attack_identifier = entry.attack_identifier @@ -230,6 +230,11 @@ def get_message_piece(self) -> MessagePiece: Returns: MessagePiece: The reconstructed message piece with all its data and scores. """ + converter_ids: Optional[List[Union[ConverterIdentifier, Dict[str, str]]]] = ( + [ConverterIdentifier.from_dict(c) for c in self.converter_identifiers] + if self.converter_identifiers + else None + ) message_piece = MessagePiece( role=self.role, original_value=self.original_value, @@ -242,7 +247,7 @@ def get_message_piece(self) -> MessagePiece: labels=self.labels, prompt_metadata=self.prompt_metadata, targeted_harm_categories=self.targeted_harm_categories, - converter_identifiers=self.converter_identifiers, + converter_identifiers=converter_ids, prompt_target_identifier=self.prompt_target_identifier, attack_identifier=self.attack_identifier, original_value_data_type=self.original_value_data_type, diff --git a/pyrit/models/message_piece.py b/pyrit/models/message_piece.py index d0dc3e6df..af16fe612 100644 --- a/pyrit/models/message_piece.py +++ b/pyrit/models/message_piece.py @@ -9,7 +9,7 @@ from uuid import uuid4 from pyrit.common.deprecation import print_deprecation_message -from pyrit.identifiers import ScorerIdentifier +from pyrit.identifiers import ConverterIdentifier, ScorerIdentifier from pyrit.models.literals import ChatMessageRole, PromptDataType, PromptResponseError from pyrit.models.score import Score @@ -38,7 +38,7 @@ def __init__( sequence: int = -1, labels: Optional[Dict[str, str]] = None, prompt_metadata: Optional[Dict[str, Union[str, int]]] = None, - converter_identifiers: Optional[List[Dict[str, str]]] = None, + converter_identifiers: Optional[List[Union[ConverterIdentifier, Dict[str, str]]]] = None, prompt_target_identifier: Optional[Dict[str, str]] = None, attack_identifier: Optional[Dict[str, str]] = None, scorer_identifier: Optional[Union[ScorerIdentifier, Dict[str, str]]] = None, @@ -69,7 +69,8 @@ def __init__( Because memory is how components talk with each other, this can be component specific. e.g. the URI from a file uploaded to a blob store, or a document type you want to upload. Defaults to None. - converter_identifiers: The converter identifiers for the prompt. Defaults to None. + converter_identifiers: The converter identifiers for the prompt. Can be ConverterIdentifier + objects or dicts (deprecated, will be removed in 0.14.0). Defaults to None. prompt_target_identifier: The target identifier for the prompt. Defaults to None. attack_identifier: The attack identifier for the prompt. Defaults to None. scorer_identifier: The scorer identifier for the prompt. Can be a ScorerIdentifier or a @@ -106,7 +107,19 @@ def __init__( self.labels = labels or {} self.prompt_metadata = prompt_metadata or {} - self.converter_identifiers = converter_identifiers if converter_identifiers else [] + # Handle converter_identifiers: convert dicts to ConverterIdentifier with deprecation warning + self.converter_identifiers: List[ConverterIdentifier] = [] + if converter_identifiers: + for conv_id in converter_identifiers: + if isinstance(conv_id, dict): + print_deprecation_message( + old_item="dict for converter_identifiers", + new_item="ConverterIdentifier", + removed_in="0.14.0", + ) + self.converter_identifiers.append(ConverterIdentifier.from_dict(conv_id)) + else: + self.converter_identifiers.append(conv_id) self.prompt_target_identifier = prompt_target_identifier or {} self.attack_identifier = attack_identifier or {} @@ -278,7 +291,7 @@ def to_dict(self) -> dict[str, object]: "labels": self.labels, "targeted_harm_categories": self.targeted_harm_categories if self.targeted_harm_categories else None, "prompt_metadata": self.prompt_metadata, - "converter_identifiers": self.converter_identifiers, + "converter_identifiers": [conv.to_dict() for conv in self.converter_identifiers], "prompt_target_identifier": self.prompt_target_identifier, "attack_identifier": self.attack_identifier, "scorer_identifier": self.scorer_identifier.to_dict() if self.scorer_identifier else None, diff --git a/pyrit/prompt_converter/add_image_text_converter.py b/pyrit/prompt_converter/add_image_text_converter.py index ca417c183..20d3ce326 100644 --- a/pyrit/prompt_converter/add_image_text_converter.py +++ b/pyrit/prompt_converter/add_image_text_converter.py @@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont from PIL.ImageFont import FreeTypeFont +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -63,6 +64,24 @@ def __init__( self._x_pos = x_pos self._y_pos = y_pos + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with image and text parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "img_to_add_path": str(self._img_to_add), + "font_name": self._font_name, + "color": self._color, + "font_size": self._font_size, + "x_pos": self._x_pos, + "y_pos": self._y_pos, + }, + ) + def _load_font(self) -> FreeTypeFont: """ Load the font for a given font name and font size. diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index 77c0347f0..dd71c5104 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -9,6 +9,7 @@ import numpy as np from pyrit.common.path import DB_DATA_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -61,6 +62,21 @@ def __init__( self._img_resize_size = img_resize_size self._video_path = video_path + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with video converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "video_path": str(self._video_path), + "img_position": self._img_position, + "img_resize_size": self._img_resize_size, + } + ) + async def _add_image_to_video(self, image_path: str, output_path: str) -> str: """ Add an image to video. diff --git a/pyrit/prompt_converter/add_text_image_converter.py b/pyrit/prompt_converter/add_text_image_converter.py index fbf121cc4..fdf9039cf 100644 --- a/pyrit/prompt_converter/add_text_image_converter.py +++ b/pyrit/prompt_converter/add_text_image_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import base64 +import hashlib import logging import string import textwrap @@ -11,6 +12,7 @@ from PIL import Image, ImageDraw, ImageFont from PIL.ImageFont import FreeTypeFont +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -63,6 +65,25 @@ def __init__( self._x_pos = x_pos self._y_pos = y_pos + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with text and image parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + text_hash = hashlib.sha256(self._text_to_add.encode("utf-8")).hexdigest()[:16] + return self._create_identifier( + converter_specific_params={ + "text_to_add_hash": text_hash, + "font_name": self._font_name, + "color": self._color, + "font_size": self._font_size, + "x_pos": self._x_pos, + "y_pos": self._y_pos, + }, + ) + def _load_font(self) -> FreeTypeFont: """ Load the font for a given font name and font size. diff --git a/pyrit/prompt_converter/ascii_art_converter.py b/pyrit/prompt_converter/ascii_art_converter.py index be8fc29b2..eae28f531 100644 --- a/pyrit/prompt_converter/ascii_art_converter.py +++ b/pyrit/prompt_converter/ascii_art_converter.py @@ -3,6 +3,7 @@ from art import text2art +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -22,7 +23,20 @@ def __init__(self, font: str = "rand") -> None: Args: font (str): The font to use for ASCII art. Defaults to "rand" which selects a random font. """ - self.font_value = font + self._font = font + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with font parameter. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "font": self._font, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -41,4 +55,4 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - return ConverterResult(output_text=text2art(prompt, font=self.font_value), output_type="text") + return ConverterResult(output_text=text2art(prompt, font=self._font), output_type="text") diff --git a/pyrit/prompt_converter/atbash_converter.py b/pyrit/prompt_converter/atbash_converter.py index d8ee91ba8..f133b9233 100644 --- a/pyrit/prompt_converter/atbash_converter.py +++ b/pyrit/prompt_converter/atbash_converter.py @@ -5,6 +5,7 @@ import string from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -39,6 +40,19 @@ def __init__(self, *, append_description: bool = False) -> None: "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with Atbash cipher parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "append_description": self.append_description, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the Atbash cipher. diff --git a/pyrit/prompt_converter/audio_frequency_converter.py b/pyrit/prompt_converter/audio_frequency_converter.py index 33153a13d..ecd74f028 100644 --- a/pyrit/prompt_converter/audio_frequency_converter.py +++ b/pyrit/prompt_converter/audio_frequency_converter.py @@ -8,6 +8,7 @@ import numpy as np from scipy.io import wavfile +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -42,6 +43,20 @@ def __init__( self._output_format = output_format self._shift_value = shift_value + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with audio frequency parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "output_format": self._output_format, + "shift_value": self._shift_value, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "audio_path") -> ConverterResult: """ Convert the given audio file by shifting its frequency. diff --git a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py index 77352e589..1b9521f65 100644 --- a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py +++ b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py @@ -10,6 +10,7 @@ from pyrit.auth.azure_auth import get_speech_config from pyrit.common import default_values +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -86,6 +87,19 @@ def __init__( # Create a flag to indicate when recognition is finished self.done = False + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with speech recognition parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "recognition_language": self._recognition_language, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "audio_path") -> ConverterResult: """ Convert the given audio file into its text representation. diff --git a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py index 87db4b71d..105177c02 100644 --- a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py +++ b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py @@ -9,6 +9,7 @@ from pyrit.auth.azure_auth import get_speech_config from pyrit.common import default_values +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -93,6 +94,21 @@ def __init__( self._synthesis_voice_name = synthesis_voice_name self._output_format = output_format + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with speech synthesis parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "synthesis_language": self._synthesis_language, + "synthesis_voice_name": self._synthesis_voice_name, + "output_format": self._output_format, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given text prompt into its audio representation. diff --git a/pyrit/prompt_converter/base64_converter.py b/pyrit/prompt_converter/base64_converter.py index 24105fe0d..ccbb685fa 100644 --- a/pyrit/prompt_converter/base64_converter.py +++ b/pyrit/prompt_converter/base64_converter.py @@ -5,6 +5,7 @@ import binascii from typing import Literal +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -44,6 +45,19 @@ def __init__(self, *, encoding_func: EncodingFunc = "b64encode") -> None: """ self._encoding_func = encoding_func + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with encoding function. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "encoding_func": self._encoding_func, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to base64 encoding. diff --git a/pyrit/prompt_converter/bin_ascii_converter.py b/pyrit/prompt_converter/bin_ascii_converter.py index 84c71f291..a3c3eac20 100644 --- a/pyrit/prompt_converter/bin_ascii_converter.py +++ b/pyrit/prompt_converter/bin_ascii_converter.py @@ -4,6 +4,7 @@ import binascii from typing import Literal, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import ( AllWordsSelectionStrategy, WordSelectionStrategy, @@ -57,6 +58,17 @@ def __init__( self._encoding_func = encoding_func + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with BinAscii converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["encoding_func"] = self._encoding_func + return self._create_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a word using the specified encoding function. diff --git a/pyrit/prompt_converter/binary_converter.py b/pyrit/prompt_converter/binary_converter.py index b1e6f5fe7..245da36f7 100644 --- a/pyrit/prompt_converter/binary_converter.py +++ b/pyrit/prompt_converter/binary_converter.py @@ -6,6 +6,7 @@ from enum import Enum from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -46,6 +47,17 @@ def __init__( raise TypeError("bits_per_char must be an instance of BinaryConverter.BitsPerChar Enum.") self.bits_per_char = bits_per_char + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with binary converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["bits_per_char"] = self.bits_per_char.value + return self._create_identifier(converter_specific_params=base_params) + def validate_input(self, prompt: str) -> None: """ Check if ``bits_per_char`` is sufficient for the characters in the prompt. diff --git a/pyrit/prompt_converter/caesar_converter.py b/pyrit/prompt_converter/caesar_converter.py index b28b0ad3e..b4a7e3816 100644 --- a/pyrit/prompt_converter/caesar_converter.py +++ b/pyrit/prompt_converter/caesar_converter.py @@ -5,6 +5,7 @@ import string from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -45,6 +46,20 @@ def __init__(self, *, caesar_offset: int, append_description: bool = False) -> N "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with Caesar cipher parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "caesar_offset": self.caesar_offset, + "append_description": self.append_description, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the Caesar cipher. diff --git a/pyrit/prompt_converter/charswap_attack_converter.py b/pyrit/prompt_converter/charswap_attack_converter.py index 1202f4493..06534f618 100644 --- a/pyrit/prompt_converter/charswap_attack_converter.py +++ b/pyrit/prompt_converter/charswap_attack_converter.py @@ -5,6 +5,7 @@ import string from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import ( WordProportionSelectionStrategy, WordSelectionStrategy, @@ -47,7 +48,20 @@ def __init__( if max_iterations <= 0: raise ValueError("max_iterations must be greater than 0") - self.max_iterations = max_iterations + self._max_iterations = max_iterations + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with charswap parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "max_iterations": self._max_iterations, + }, + ) async def convert_word_async(self, word: str) -> str: """ @@ -73,7 +87,7 @@ def _perturb_word(self, word: str) -> str: """ if word not in string.punctuation and len(word) > 3: idx_elements = list(word) - for _ in range(self.max_iterations): + for _ in range(self._max_iterations): idx1 = random.randint(1, len(word) - 2) # Swap characters idx_elements[idx1], idx_elements[idx1 + 1] = ( diff --git a/pyrit/prompt_converter/codechameleon_converter.py b/pyrit/prompt_converter/codechameleon_converter.py index cb4b8af1d..c51db730d 100644 --- a/pyrit/prompt_converter/codechameleon_converter.py +++ b/pyrit/prompt_converter/codechameleon_converter.py @@ -9,6 +9,7 @@ from typing import Any, Callable, Optional from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -98,6 +99,21 @@ def __init__( '"reverse", "binary_tree", "odd_even" or "length".' ) + self._encrypt_type = encrypt_type + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with encryption type. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "encrypt_type": self._encrypt_type, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by applying the specified encryption function. diff --git a/pyrit/prompt_converter/colloquial_wordswap_converter.py b/pyrit/prompt_converter/colloquial_wordswap_converter.py index 7fd5c842d..96ab2d6c1 100644 --- a/pyrit/prompt_converter/colloquial_wordswap_converter.py +++ b/pyrit/prompt_converter/colloquial_wordswap_converter.py @@ -5,6 +5,7 @@ import re from typing import Dict, List, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -51,6 +52,20 @@ def __init__( self._colloquial_substitutions = custom_substitutions if custom_substitutions else default_substitutions self._deterministic = deterministic + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with colloquial wordswap parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "deterministic": self._deterministic, + "substitution_keys": sorted(self._colloquial_substitutions.keys()), + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by replacing words with colloquial Singaporean terms. diff --git a/pyrit/prompt_converter/diacritic_converter.py b/pyrit/prompt_converter/diacritic_converter.py index 64ebaa753..08e229e16 100644 --- a/pyrit/prompt_converter/diacritic_converter.py +++ b/pyrit/prompt_converter/diacritic_converter.py @@ -4,6 +4,7 @@ import logging import unicodedata +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -43,6 +44,20 @@ def __init__(self, target_chars: str = "aeiou", accent: str = "acute"): self._target_chars = set(target_chars) self._accent = accent + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with diacritic parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "target_chars": sorted(self._target_chars), + "accent": self._accent, + } + ) + def _get_accent_mark(self) -> str: """ Retrieve the Unicode character for the specified diacritic accent. diff --git a/pyrit/prompt_converter/first_letter_converter.py b/pyrit/prompt_converter/first_letter_converter.py index b3bc03266..7449fc8f2 100644 --- a/pyrit/prompt_converter/first_letter_converter.py +++ b/pyrit/prompt_converter/first_letter_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -30,6 +31,17 @@ def __init__( super().__init__(word_selection_strategy=word_selection_strategy, word_split_separator=None) self.letter_separator = letter_separator + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with first letter converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["letter_separator"] = self.letter_separator + return self._create_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a single word into the target format supported by the converter. diff --git a/pyrit/prompt_converter/human_in_the_loop_converter.py b/pyrit/prompt_converter/human_in_the_loop_converter.py index 72398cb6f..249497242 100644 --- a/pyrit/prompt_converter/human_in_the_loop_converter.py +++ b/pyrit/prompt_converter/human_in_the_loop_converter.py @@ -4,6 +4,7 @@ import logging from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -33,6 +34,15 @@ def __init__( """ self._converters = converters or [] + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with sub-converters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier(sub_converters=self._converters) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by allowing user interaction before sending it to a target. diff --git a/pyrit/prompt_converter/image_compression_converter.py b/pyrit/prompt_converter/image_compression_converter.py index aa4995695..916e8ebee 100644 --- a/pyrit/prompt_converter/image_compression_converter.py +++ b/pyrit/prompt_converter/image_compression_converter.py @@ -10,6 +10,7 @@ import aiohttp from PIL import Image +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -123,6 +124,25 @@ def __init__( "Using quality > 95 for JPEG may result in larger files. Consider using a lower quality setting." ) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with image compression parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "output_format": self._output_format, + "quality": self._quality, + "optimize": self._optimize, + "progressive": self._progressive, + "compress_level": self._compress_level, + "lossless": self._lossless, + "method": self._method, + } + ) + def _should_compress(self, original_size: int) -> bool: """ Determine if image should be compressed. diff --git a/pyrit/prompt_converter/insert_punctuation_converter.py b/pyrit/prompt_converter/insert_punctuation_converter.py index 885fb64da..91d15378d 100644 --- a/pyrit/prompt_converter/insert_punctuation_converter.py +++ b/pyrit/prompt_converter/insert_punctuation_converter.py @@ -6,6 +6,7 @@ import string from typing import List, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -44,6 +45,20 @@ def __init__(self, word_swap_ratio: float = 0.2, between_words: bool = True) -> self._word_swap_ratio = word_swap_ratio self._between_words = between_words + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with punctuation insertion parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "word_swap_ratio": self._word_swap_ratio, + "between_words": self._between_words, + } + ) + def _is_valid_punctuation(self, punctuation_list: List[str]) -> bool: """ Check if all items in the list are valid punctuation characters in string.punctuation. diff --git a/pyrit/prompt_converter/leetspeak_converter.py b/pyrit/prompt_converter/leetspeak_converter.py index 913bb2110..5d8cc8c28 100644 --- a/pyrit/prompt_converter/leetspeak_converter.py +++ b/pyrit/prompt_converter/leetspeak_converter.py @@ -4,6 +4,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -49,6 +50,30 @@ def __init__( # Use custom substitutions if provided, otherwise default to the standard ones self._leet_substitutions = custom_substitutions if custom_substitutions else default_substitutions self._deterministic = deterministic + self._has_custom_substitutions = custom_substitutions is not None + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with leetspeak parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + import hashlib + import json + + # Hash custom substitutions if provided + substitutions_hash = None + if self._has_custom_substitutions: + substitutions_str = json.dumps(self._leet_substitutions, sort_keys=True) + substitutions_hash = hashlib.sha256(substitutions_str.encode("utf-8")).hexdigest()[:16] + + return self._create_identifier( + converter_specific_params={ + "deterministic": self._deterministic, + "custom_substitutions_hash": substitutions_hash, + }, + ) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/llm_generic_text_converter.py b/pyrit/prompt_converter/llm_generic_text_converter.py index 8a8a3a361..b03be4f38 100644 --- a/pyrit/prompt_converter/llm_generic_text_converter.py +++ b/pyrit/prompt_converter/llm_generic_text_converter.py @@ -1,11 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import hashlib import logging import uuid from typing import Any, Optional from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -61,6 +63,34 @@ def __init__( self._user_prompt_template_with_objective = user_prompt_template_with_objective + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with LLM and template parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + # Hash templates if they exist and have a value attribute + system_prompt_hash = None + if self._system_prompt_template and hasattr(self._system_prompt_template, "value"): + system_prompt_hash = hashlib.sha256(str(self._system_prompt_template.value).encode("utf-8")).hexdigest()[ + :16 + ] + + user_prompt_hash = None + if self._user_prompt_template_with_objective and hasattr(self._user_prompt_template_with_objective, "value"): + user_prompt_hash = hashlib.sha256( + str(self._user_prompt_template_with_objective.value).encode("utf-8") + ).hexdigest()[:16] + + return self._create_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "system_prompt_template_hash": system_prompt_hash, + "user_prompt_template_hash": user_prompt_hash, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using an LLM via the specified converter target. diff --git a/pyrit/prompt_converter/math_obfuscation_converter.py b/pyrit/prompt_converter/math_obfuscation_converter.py index cc4224d8c..83c559ba9 100644 --- a/pyrit/prompt_converter/math_obfuscation_converter.py +++ b/pyrit/prompt_converter/math_obfuscation_converter.py @@ -5,6 +5,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -88,6 +89,20 @@ def __init__( self._suffix = suffix if suffix is not None else self.DEFAULT_SUFFIX self._rng = rng or random.Random() + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with math obfuscation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "min_n": self._min_n, + "max_n": self._max_n, + } + ) + async def convert_async( self, *, diff --git a/pyrit/prompt_converter/morse_converter.py b/pyrit/prompt_converter/morse_converter.py index ff8548b9e..8b837258f 100644 --- a/pyrit/prompt_converter/morse_converter.py +++ b/pyrit/prompt_converter/morse_converter.py @@ -4,6 +4,7 @@ import pathlib from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -34,6 +35,19 @@ def __init__(self, *, append_description: bool = False) -> None: "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with morse converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "append_description": self.append_description, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to morse code. diff --git a/pyrit/prompt_converter/noise_converter.py b/pyrit/prompt_converter/noise_converter.py index a3c2a3b55..5496e967f 100644 --- a/pyrit/prompt_converter/noise_converter.py +++ b/pyrit/prompt_converter/noise_converter.py @@ -8,6 +8,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -60,3 +61,20 @@ def __init__( noise=noise, number_errors=str(number_errors), ) + self._noise = noise + self._number_errors = number_errors + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with noise parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "noise": self._noise, + "number_errors": self._number_errors, + }, + ) diff --git a/pyrit/prompt_converter/pdf_converter.py b/pyrit/prompt_converter/pdf_converter.py index 9ff4fab80..b84cb23dd 100644 --- a/pyrit/prompt_converter/pdf_converter.py +++ b/pyrit/prompt_converter/pdf_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import ast +import hashlib from io import BytesIO from pathlib import Path from typing import Any, Dict, List, Optional @@ -12,6 +13,7 @@ from reportlab.pdfgen import canvas from pyrit.common.logger import logger +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt, data_serializer_factory from pyrit.models.data_type_serializer import DataTypeSerializer from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -103,6 +105,32 @@ def __init__( if not all(isinstance(item, dict) for item in self._injection_items): raise ValueError("Each injection item must be a dictionary.") + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with PDF converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + template_hash = None + if self._prompt_template: + template_hash = hashlib.sha256(str(self._prompt_template.value).encode("utf-8")).hexdigest()[:16] + + existing_pdf_path = None + if self._existing_pdf_path: + existing_pdf_path = str(self._existing_pdf_path) + + return self._create_identifier( + converter_specific_params={ + "font_type": self._font_type, + "font_size": self._font_size, + "page_width": self._page_width, + "page_height": self._page_height, + "prompt_template_hash": template_hash, + "existing_pdf_path": existing_pdf_path, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt into a PDF. diff --git a/pyrit/prompt_converter/persuasion_converter.py b/pyrit/prompt_converter/persuasion_converter.py index 4321ff0eb..7282b43f1 100644 --- a/pyrit/prompt_converter/persuasion_converter.py +++ b/pyrit/prompt_converter/persuasion_converter.py @@ -13,6 +13,7 @@ pyrit_json_retry, remove_markdown_json, ) +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -77,6 +78,21 @@ def __init__( except FileNotFoundError: raise ValueError(f"Persuasion technique '{persuasion_technique}' does not exist or is not supported.") self.system_prompt = str(prompt_template.value) + self._persuasion_technique = persuasion_technique + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with persuasion parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self.converter_target, + converter_specific_params={ + "persuasion_technique": self._persuasion_technique, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ diff --git a/pyrit/prompt_converter/prompt_converter.py b/pyrit/prompt_converter/prompt_converter.py index d83f9a043..f7e0a6a2c 100644 --- a/pyrit/prompt_converter/prompt_converter.py +++ b/pyrit/prompt_converter/prompt_converter.py @@ -6,10 +6,10 @@ import inspect import re from dataclasses import dataclass -from typing import get_args +from typing import Any, Dict, List, Optional, Sequence, get_args from pyrit import prompt_converter -from pyrit.identifiers import LegacyIdentifiable +from pyrit.identifiers import ConverterIdentifier, Identifiable from pyrit.models import PromptDataType @@ -32,7 +32,7 @@ def __str__(self) -> str: return f"{self.output_type}: {self.output_text}" -class PromptConverter(LegacyIdentifiable): +class PromptConverter(Identifiable[ConverterIdentifier]): """ Base class for converters that transform prompts into a different representation or format. @@ -48,6 +48,8 @@ class PromptConverter(LegacyIdentifiable): #: Tuple of output modalities supported by this converter. Subclasses must override this. SUPPORTED_OUTPUT_TYPES: tuple[PromptDataType, ...] = () + _identifier: Optional[ConverterIdentifier] = None + def __init_subclass__(cls, **kwargs: object) -> None: """ Validate that concrete subclasses define required class attributes. @@ -163,17 +165,67 @@ async def _replace_text_match(self, match: str) -> ConverterResult: result = await self.convert_async(prompt=match, input_type="text") return result - def get_identifier(self) -> dict[str, str]: + def _build_identifier(self) -> ConverterIdentifier: """ - Return an identifier dictionary for the converter. + Build and return the identifier for this converter. + + Subclasses can override this method to add converter-specific parameters + by calling _create_identifier with additional arguments. + + The default implementation calls _create_identifier with no extra parameters. + + Returns: + ConverterIdentifier: The constructed identifier. + """ + return self._create_identifier() + + def _create_identifier( + self, + *, + sub_converters: Optional[Sequence["PromptConverter"]] = None, + converter_target: Optional[Any] = None, + converter_specific_params: Optional[Dict[str, Any]] = None, + ) -> ConverterIdentifier: + """ + Construct and return the converter identifier. + + Args: + sub_converters: List of sub-converters for composite converters + (e.g., ConverterPipeline). Defaults to None. + converter_target: The prompt target used by this converter (for LLM-based converters). + Defaults to None. + converter_specific_params: Additional converter-specific parameters. + Defaults to None. Returns: - dict: The identifier dictionary. + ConverterIdentifier: The constructed identifier. """ - public_attributes = {} - public_attributes["__type__"] = self.__class__.__name__ - public_attributes["__module__"] = self.__class__.__module__ - return public_attributes + # Build sub_identifier from sub_converters + sub_identifier: Optional[List[ConverterIdentifier]] = None + if sub_converters: + sub_identifier = [converter.get_identifier() for converter in sub_converters] + + # Extract target_info from converter_target + target_info: Optional[Dict[str, Any]] = None + if converter_target: + target_id = converter_target.get_identifier() + # Extract standard fields for converter identification + target_info = {} + for key in ["__type__", "model_name", "temperature", "top_p"]: + if key in target_id: + target_info[key] = target_id[key] + + return ConverterIdentifier( + class_name=self.__class__.__name__, + class_module=self.__class__.__module__, + class_description=self.__class__.__doc__ or "", + identifier_type="instance", + supported_input_types=self.SUPPORTED_INPUT_TYPES, + supported_output_types=self.SUPPORTED_OUTPUT_TYPES, + sub_identifier=sub_identifier, + target_info=target_info, + converter_specific_params=converter_specific_params, + ) @property def supported_input_types(self) -> list[PromptDataType]: diff --git a/pyrit/prompt_converter/qr_code_converter.py b/pyrit/prompt_converter/qr_code_converter.py index 10e20f771..d4e9c7ea8 100644 --- a/pyrit/prompt_converter/qr_code_converter.py +++ b/pyrit/prompt_converter/qr_code_converter.py @@ -5,6 +5,7 @@ import segno +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -60,6 +61,22 @@ def __init__( self._border_color = border_color or light_color self._img_serializer = data_serializer_factory(category="prompt-memory-entries", data_type="image_path") + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with QR code parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "scale": self._scale, + "border": self._border, + "dark_color": self._dark_color, + "light_color": self._light_color, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to a QR code image. diff --git a/pyrit/prompt_converter/random_capital_letters_converter.py b/pyrit/prompt_converter/random_capital_letters_converter.py index a86f4b8a9..9e87a30e7 100644 --- a/pyrit/prompt_converter/random_capital_letters_converter.py +++ b/pyrit/prompt_converter/random_capital_letters_converter.py @@ -4,6 +4,7 @@ import logging import random +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -26,6 +27,19 @@ def __init__(self, percentage: float = 100.0) -> None: """ self.percentage = percentage + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with random capital letters parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "percentage": self.percentage, + } + ) + def is_percentage(self, input_string: float) -> bool: """ Check if the input string is a valid percentage between 1 and 100. diff --git a/pyrit/prompt_converter/repeat_token_converter.py b/pyrit/prompt_converter/repeat_token_converter.py index 43d2548dc..e568c2514 100644 --- a/pyrit/prompt_converter/repeat_token_converter.py +++ b/pyrit/prompt_converter/repeat_token_converter.py @@ -4,6 +4,7 @@ import re from typing import Literal, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -46,8 +47,9 @@ def __init__( token_insert_mode (str, optional): The mode of insertion for the repeated token. Can be "split", "prepend", "append", or "repeat". """ - self.token_to_repeat = " " + token_to_repeat.strip() - self.times_to_repeat = times_to_repeat + self._token_to_repeat = " " + token_to_repeat.strip() + self._times_to_repeat = times_to_repeat + self._token_insert_mode = token_insert_mode if token_insert_mode else "split" if not token_insert_mode: token_insert_mode = "split" @@ -80,6 +82,21 @@ def insert(text: str) -> list[str]: self.insert = insert + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with repeat token parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "token_to_repeat": self._token_to_repeat.strip(), + "times_to_repeat": self._times_to_repeat, + "token_insert_mode": self._token_insert_mode, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by repeating the specified token a specified number of times. @@ -99,6 +116,6 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text prompt_parts = self.insert(prompt) return ConverterResult( - output_text=f"{prompt_parts[0]}{self.token_to_repeat * self.times_to_repeat}{prompt_parts[1]}", + output_text=f"{prompt_parts[0]}{self._token_to_repeat * self._times_to_repeat}{prompt_parts[1]}", output_type="text", ) diff --git a/pyrit/prompt_converter/search_replace_converter.py b/pyrit/prompt_converter/search_replace_converter.py index 5f1ec8feb..4439cc66f 100644 --- a/pyrit/prompt_converter/search_replace_converter.py +++ b/pyrit/prompt_converter/search_replace_converter.py @@ -4,6 +4,7 @@ import random import re +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -26,10 +27,23 @@ def __init__(self, pattern: str, replace: str | list[str], regex_flags: int = 0) If a list is provided, a random element will be chosen for replacement. regex_flags (int): Regex flags to use for the replacement. Defaults to 0 (no flags). """ - self.pattern = pattern - self.replace_list = [replace] if isinstance(replace, str) else replace + self._pattern = pattern + self._replace_list = [replace] if isinstance(replace, str) else replace + self._regex_flags = regex_flags - self.regex_flags = regex_flags + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with search/replace parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "pattern": self._pattern, + "replace_list": self._replace_list, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -48,8 +62,8 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - replace = random.choice(self.replace_list) + replace = random.choice(self._replace_list) return ConverterResult( - output_text=re.sub(self.pattern, replace, prompt, flags=self.regex_flags), output_type="text" + output_text=re.sub(self._pattern, replace, prompt, flags=self._regex_flags), output_type="text" ) diff --git a/pyrit/prompt_converter/selective_text_converter.py b/pyrit/prompt_converter/selective_text_converter.py index 093348ddd..393606d96 100644 --- a/pyrit/prompt_converter/selective_text_converter.py +++ b/pyrit/prompt_converter/selective_text_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter from pyrit.prompt_converter.text_selection_strategy import ( @@ -89,6 +90,23 @@ def __init__( self._is_word_level = isinstance(selection_strategy, WordSelectionStrategy) self._is_token_based = isinstance(selection_strategy, TokenSelectionStrategy) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with selective text converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + sub_converters=[self._converter], + converter_specific_params={ + "selection_strategy": self._selection_strategy.__class__.__name__, + "preserve_tokens": self._preserve_tokens, + "start_token": self._start_token, + "end_token": self._end_token, + }, + ) + def _validate_converter( self, *, diff --git a/pyrit/prompt_converter/string_join_converter.py b/pyrit/prompt_converter/string_join_converter.py index 3122bf02b..ab94c8517 100644 --- a/pyrit/prompt_converter/string_join_converter.py +++ b/pyrit/prompt_converter/string_join_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -27,7 +28,20 @@ def __init__( If None, all words will be converted. """ super().__init__(word_selection_strategy=word_selection_strategy) - self.join_value = join_value + self._join_value = join_value + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with join parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "join_value": self._join_value, + }, + ) async def convert_word_async(self, word: str) -> str: """ @@ -39,4 +53,4 @@ async def convert_word_async(self, word: str) -> str: Returns: str: The converted word. """ - return self.join_value.join(word) + return self._join_value.join(word) diff --git a/pyrit/prompt_converter/suffix_append_converter.py b/pyrit/prompt_converter/suffix_append_converter.py index 86a8c4323..3aff3fda4 100644 --- a/pyrit/prompt_converter/suffix_append_converter.py +++ b/pyrit/prompt_converter/suffix_append_converter.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -29,7 +30,20 @@ def __init__(self, *, suffix: str): if not suffix: raise ValueError("Please specify a suffix (str) to be appended to the prompt.") - self.suffix = suffix + self._suffix = suffix + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with suffix parameter. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "suffix": self._suffix, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -48,4 +62,4 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - return ConverterResult(output_text=prompt + " " + self.suffix, output_type="text") + return ConverterResult(output_text=prompt + " " + self._suffix, output_type="text") diff --git a/pyrit/prompt_converter/template_segment_converter.py b/pyrit/prompt_converter/template_segment_converter.py index 737275108..61defff6d 100644 --- a/pyrit/prompt_converter/template_segment_converter.py +++ b/pyrit/prompt_converter/template_segment_converter.py @@ -1,12 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import hashlib import logging import pathlib import random from typing import Optional from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -69,6 +71,21 @@ def __init__( f"Template parameters: {self.prompt_template.parameters}" ) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with template parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + template_hash = hashlib.sha256(str(self.prompt_template.value).encode("utf-8")).hexdigest()[:16] + return self._create_identifier( + converter_specific_params={ + "template_hash": template_hash, + "number_parameters": self._number_parameters, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by splitting it into random segments and using them to fill the template parameters. diff --git a/pyrit/prompt_converter/tense_converter.py b/pyrit/prompt_converter/tense_converter.py index 32956a865..13e2af822 100644 --- a/pyrit/prompt_converter/tense_converter.py +++ b/pyrit/prompt_converter/tense_converter.py @@ -7,6 +7,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -50,3 +51,18 @@ def __init__( system_prompt_template=prompt_template, tense=tense, ) + self._tense = tense + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with tense parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "tense": self._tense, + }, + ) diff --git a/pyrit/prompt_converter/text_jailbreak_converter.py b/pyrit/prompt_converter/text_jailbreak_converter.py index 3e8f5c969..84d03a6c6 100644 --- a/pyrit/prompt_converter/text_jailbreak_converter.py +++ b/pyrit/prompt_converter/text_jailbreak_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. from pyrit.datasets import TextJailBreak +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -23,6 +24,19 @@ def __init__(self, *, jailbreak_template: TextJailBreak): """ self.jail_break_template = jailbreak_template + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with jailbreak template path. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "jailbreak_template_path": self.jail_break_template.template_source, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the jailbreak template. diff --git a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py index 3169eeec1..b90440953 100644 --- a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -32,6 +33,17 @@ def __init__(self, action: Literal["encode", "decode"] = "encode", unicode_tags: self.unicode_tags = unicode_tags super().__init__(action=action) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with ASCII smuggler parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["unicode_tags"] = self.unicode_tags + return self._create_identifier(converter_specific_params=base_params) + def encode_message(self, *, message: str) -> tuple[str, str]: """ Encode the message using Unicode Tags. diff --git a/pyrit/prompt_converter/token_smuggling/base.py b/pyrit/prompt_converter/token_smuggling/base.py index 1c4d00798..f41d18473 100644 --- a/pyrit/prompt_converter/token_smuggling/base.py +++ b/pyrit/prompt_converter/token_smuggling/base.py @@ -5,6 +5,7 @@ import logging from typing import Literal, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -36,6 +37,19 @@ def __init__(self, action: Literal["encode", "decode"] = "encode") -> None: raise ValueError("Action must be either 'encode' or 'decode'") self.action = action + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with smuggler action. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "action": self.action, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by either encoding or decoding it based on the specified action. diff --git a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py index 4e9ea1947..26edf77a7 100644 --- a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal, Optional, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -42,6 +43,18 @@ def __init__( self.zero_char = zero_char if zero_char is not None else "\u2062" # Invisible Times self.one_char = one_char if one_char is not None else "\u2064" # Invisible Plus + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with sneaky bits parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["zero_char_codepoint"] = hex(ord(self.zero_char)) + base_params["one_char_codepoint"] = hex(ord(self.one_char)) + return self._create_identifier(converter_specific_params=base_params) + def encode_message(self, message: str) -> Tuple[str, str]: """ Encode the message using Sneaky Bits mode. diff --git a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py index ed6ad68ac..1d3b5ef26 100644 --- a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal, Optional, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -51,6 +52,18 @@ def __init__( self.utf8_base_char = base_char_utf8 if base_char_utf8 is not None else "😊" self.embed_in_base = embed_in_base + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with variation selector parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["base_char"] = self.utf8_base_char + base_params["embed_in_base"] = self.embed_in_base + return self._create_identifier(converter_specific_params=base_params) + def encode_message(self, message: str) -> Tuple[str, str]: """ Encode the message using Unicode variation selectors. diff --git a/pyrit/prompt_converter/tone_converter.py b/pyrit/prompt_converter/tone_converter.py index 9b2039846..2c55f1ded 100644 --- a/pyrit/prompt_converter/tone_converter.py +++ b/pyrit/prompt_converter/tone_converter.py @@ -7,6 +7,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -53,3 +54,18 @@ def __init__( system_prompt_template=prompt_template, tone=tone, ) + self._tone = tone + + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with tone parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "tone": self._tone, + }, + ) diff --git a/pyrit/prompt_converter/translation_converter.py b/pyrit/prompt_converter/translation_converter.py index 4c44d80cd..cf6dba7cb 100644 --- a/pyrit/prompt_converter/translation_converter.py +++ b/pyrit/prompt_converter/translation_converter.py @@ -16,6 +16,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -81,6 +82,20 @@ def __init__( self.system_prompt = prompt_template.render_template_value(languages=language) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with translation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self.converter_target, + converter_specific_params={ + "language": self.language, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by translating it using the converter target. diff --git a/pyrit/prompt_converter/transparency_attack_converter.py b/pyrit/prompt_converter/transparency_attack_converter.py index e39b17801..419d58572 100644 --- a/pyrit/prompt_converter/transparency_attack_converter.py +++ b/pyrit/prompt_converter/transparency_attack_converter.py @@ -10,6 +10,7 @@ import numpy from PIL import Image +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -184,6 +185,22 @@ def __init__( self._cached_benign_image = self._load_and_preprocess_image(str(benign_image_path)) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with transparency attack parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "benign_image_path": str(self.benign_image_path), + "size": self.size, + "steps": self.steps, + "learning_rate": self.learning_rate, + } + ) + def _load_and_preprocess_image(self, path: str) -> numpy.ndarray: # type: ignore[type-arg, unused-ignore] """ Load image, convert to grayscale, resize, and normalize for optimization. diff --git a/pyrit/prompt_converter/unicode_confusable_converter.py b/pyrit/prompt_converter/unicode_confusable_converter.py index 7afd99f3c..b9d9790a7 100644 --- a/pyrit/prompt_converter/unicode_confusable_converter.py +++ b/pyrit/prompt_converter/unicode_confusable_converter.py @@ -9,6 +9,7 @@ from confusable_homoglyphs.confusables import is_confusable from confusables import confusable_characters +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -58,6 +59,20 @@ def __init__( self._source_package = source_package self._deterministic = deterministic + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with unicode confusable parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "source_package": self._source_package, + "deterministic": self._deterministic, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by applying confusable substitutions. This leads to a prompt that looks similar, diff --git a/pyrit/prompt_converter/unicode_replacement_converter.py b/pyrit/prompt_converter/unicode_replacement_converter.py index 70a8442b7..e5a3862b0 100644 --- a/pyrit/prompt_converter/unicode_replacement_converter.py +++ b/pyrit/prompt_converter/unicode_replacement_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -29,6 +30,17 @@ def __init__( super().__init__(word_selection_strategy=word_selection_strategy) self.encode_spaces = encode_spaces + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with unicode replacement parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["encode_spaces"] = self.encode_spaces + return self._create_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a single word into the target format supported by the converter. diff --git a/pyrit/prompt_converter/unicode_sub_converter.py b/pyrit/prompt_converter/unicode_sub_converter.py index 64a5054e3..d85272da8 100644 --- a/pyrit/prompt_converter/unicode_sub_converter.py +++ b/pyrit/prompt_converter/unicode_sub_converter.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -22,6 +23,19 @@ def __init__(self, *, start_value: int = 0xE0000) -> None: """ self.startValue = start_value + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with unicode substitution parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "start_value": self.startValue, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by encoding it using any unicode starting point. diff --git a/pyrit/prompt_converter/variation_converter.py b/pyrit/prompt_converter/variation_converter.py index 38ee79362..8b9ca98aa 100644 --- a/pyrit/prompt_converter/variation_converter.py +++ b/pyrit/prompt_converter/variation_converter.py @@ -15,6 +15,7 @@ pyrit_json_retry, remove_markdown_json, ) +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -67,6 +68,17 @@ def __init__( self.system_prompt = str(prompt_template.render_template_value(number_iterations=str(self.number_variations))) + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with variation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_target=self.converter_target, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by generating variations of it using the converter target. diff --git a/pyrit/prompt_converter/word_level_converter.py b/pyrit/prompt_converter/word_level_converter.py index 2241ca005..96ed8a066 100644 --- a/pyrit/prompt_converter/word_level_converter.py +++ b/pyrit/prompt_converter/word_level_converter.py @@ -4,6 +4,7 @@ import abc from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter from pyrit.prompt_converter.text_selection_strategy import ( @@ -46,6 +47,20 @@ def __init__( self._word_selection_strategy = word_selection_strategy or AllWordsSelectionStrategy() self._word_split_separator = word_split_separator + def _build_identifier(self) -> ConverterIdentifier: + """ + Build identifier with word-level converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "word_selection_strategy": self._word_selection_strategy.__class__.__name__, + "word_split_separator": self._word_split_separator, + } + ) + @abc.abstractmethod async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/zalgo_converter.py b/pyrit/prompt_converter/zalgo_converter.py index 06a7abbf3..f8a23106d 100644 --- a/pyrit/prompt_converter/zalgo_converter.py +++ b/pyrit/prompt_converter/zalgo_converter.py @@ -5,6 +5,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -40,6 +41,19 @@ def __init__( self._intensity = self._normalize_intensity(intensity) self._seed = seed + def _build_identifier(self) -> ConverterIdentifier: + """ + Build the converter identifier with zalgo parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._create_identifier( + converter_specific_params={ + "intensity": self._intensity, + }, + ) + def _normalize_intensity(self, intensity: int) -> int: try: intensity = int(intensity) diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index a4694478e..1b93088cb 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -6,6 +6,7 @@ from typing import Optional, Type, cast from uuid import UUID +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer import Scorer @@ -195,9 +196,14 @@ def _get_wrapped_scorer(self) -> Scorer: """Return the wrapped scorer.""" return self._wrapped_scorer - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this conversation scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this conversation scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=[self._wrapped_scorer], ) diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 8aab58b92..9b71f20a3 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -17,6 +17,7 @@ from pyrit.auth import TokenProviderCredential from pyrit.common import default_values +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ( DataTypeSerializer, MessagePiece, @@ -146,9 +147,14 @@ def _category_values(self) -> list[str]: """Get the string values of the configured harm categories for API calls.""" return [category.value for category in self._harm_categories] - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( scorer_specific_params={ "score_categories": self._category_values, } diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index 8d214eea0..6086493d3 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -7,6 +7,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SEED_PROMPT_PATH from pyrit.exceptions.exception_classes import InvalidJsonException +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -55,9 +56,14 @@ def __init__( # Render the system prompt with the harm category self._system_prompt = scoring_instructions_template.render_template_value(harm_categories=self._harm_category) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index 3216cfbe5..f1a36cd13 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -7,6 +7,7 @@ import numpy as np +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -55,9 +56,14 @@ def __init__( self.metric = metric self.n = n - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( scorer_specific_params={ "reference_text": self.reference_text, "metric": self.metric.value, diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index b0d53c0bf..e1d482928 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -87,9 +88,14 @@ def __init__( self._metadata_output_key = metadata_output_key self._category_output_key = category_output_key - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index a374d814c..c01fe6507 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -10,6 +10,7 @@ import yaml from pyrit.common.path import HARM_DEFINITION_PATH, SCORER_LIKERT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -189,9 +190,14 @@ def __init__( self._set_likert_scale_system_prompt(likert_scale_path=likert_scale.path) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index f6c5611bf..5e502681d 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SCALES_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -83,9 +84,14 @@ def __init__( self._system_prompt = scoring_instructions_template.render_template_value(**scale_args) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index ae5aca3bc..54c81ec1f 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -3,6 +3,7 @@ from typing import List, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_score_aggregator import ( FloatScaleAggregatorFunc, @@ -61,9 +62,14 @@ def __init__( ) self._score_aggregator = score_aggregator - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index 681636606..f9b3226ba 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -4,6 +4,7 @@ import asyncio from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -46,9 +47,14 @@ def __init__( self._rpc_server = AppRPCServer(open_browser=open_browser) self._rpc_server.start() - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index c9fe2a9d5..61c1d466e 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -95,23 +95,11 @@ def scorer_type(self) -> ScoreType: else: return "unknown" - def get_identifier(self) -> ScorerIdentifier: - """ - Get the scorer identifier. Built lazily on first access. - - Returns: - ScorerIdentifier: The identifier containing all configuration parameters. - """ - if self._identifier is None: - self._build_identifier() - assert self._identifier is not None, "_build_identifier must set _identifier" - return self._identifier - @property def _memory(self) -> MemoryInterface: return CentralMemory.get_memory_instance() - def _set_identifier( + def _create_identifier( self, *, system_prompt_template: Optional[str] = None, @@ -120,9 +108,9 @@ def _set_identifier( score_aggregator: Optional[str] = None, scorer_specific_params: Optional[Dict[str, Any]] = None, prompt_target: Optional[PromptTarget] = None, - ) -> None: + ) -> ScorerIdentifier: """ - Construct the scorer evaluation identifier. + Construct and return the scorer identifier. Args: system_prompt_template (Optional[str]): The system prompt template used by this scorer. Defaults to None. @@ -132,6 +120,9 @@ def _set_identifier( scorer_specific_params (Optional[Dict[str, Any]]): Additional scorer-specific parameters. Defaults to None. prompt_target (Optional[PromptTarget]): The prompt target used by this scorer. Defaults to None. + + Returns: + ScorerIdentifier: The constructed identifier. """ # Build sub_identifier from sub_scorers (store as dicts for storage) sub_identifier: Optional[List[ScorerIdentifier]] = None @@ -147,7 +138,7 @@ def _set_identifier( if key in target_id: target_info[key] = target_id[key] - self._identifier = ScorerIdentifier( + return ScorerIdentifier( class_name=self.__class__.__name__, class_module=self.__class__.__module__, class_description=self.__class__.__doc__ or "", diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index ebd0d4af4..8034f5a00 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -4,6 +4,7 @@ from typing import Optional from pyrit.analytics.text_matching import ExactTextMatching, TextMatching +from pyrit.identifiers import ScorerIdentifier from pyrit.memory.central_memory import CentralMemory from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -51,9 +52,14 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "text_matcher": self._text_matcher.__class__.__name__, diff --git a/pyrit/score/true_false/float_scale_threshold_scorer.py b/pyrit/score/true_false/float_scale_threshold_scorer.py index 76e0b8bec..5ca8e274c 100644 --- a/pyrit/score/true_false/float_scale_threshold_scorer.py +++ b/pyrit/score/true_false/float_scale_threshold_scorer.py @@ -4,6 +4,7 @@ import uuid from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.float_scale.float_scale_score_aggregator import ( FloatScaleAggregatorFunc, @@ -54,9 +55,14 @@ def threshold(self) -> float: """Get the threshold value used for score comparison.""" return self._threshold - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 6c6bcdf38..932bf49df 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -9,6 +9,7 @@ from openai import BadRequestError from pyrit.exceptions import PyritException, pyrit_target_retry +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score from pyrit.prompt_target import GandalfLevel, PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -54,9 +55,14 @@ def __init__( self._defender = level.value self._endpoint = "https://gandalf-api.lakera.ai/api/guess-password" - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index a4bbd8e46..f45fc90f1 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -4,6 +4,7 @@ import re from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -42,9 +43,14 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index 6d3c4946d..edfc697b1 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -6,6 +6,7 @@ import uuid from typing import Any, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score, ScoreType from pyrit.prompt_target import PromptShieldTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -48,9 +49,14 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index 514b39ad6..63ba8c6c1 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -50,9 +51,14 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "correct_answer_matching_patterns": self._correct_answer_matching_patterns, diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 3942f5396..5697d17c5 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_CONTENT_CLASSIFIERS_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -77,9 +78,14 @@ def __init__( no_category_found=self._no_category_found_category, ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index 957ef2722..edc1cc5c0 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -84,9 +85,14 @@ def __init__( self._metadata_output_key = metadata_output_key self._category_output_key = category_output_key - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index 8b3467a1f..4341bdc33 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -5,6 +5,7 @@ from typing import Optional from pyrit.common.path import SCORER_SEED_PROMPT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -71,9 +72,14 @@ def __init__( self._score_category = ["refusal"] - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt_with_objective, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 1afc533d2..5097f7be3 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SEED_PROMPT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -147,9 +148,14 @@ def __init__( true_description=true_category, false_description=false_category, metadata=metadata ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index 5dc351547..e4548378b 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -4,6 +4,7 @@ from typing import Optional from pyrit.analytics.text_matching import ExactTextMatching, TextMatching +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -50,9 +51,14 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "substring": self._substring, diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 888a36cda..5dc2bb818 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -4,6 +4,7 @@ import asyncio from typing import List, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc @@ -52,9 +53,14 @@ def __init__( self._scorers = scorers - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=self._scorers, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/true_false_inverter_scorer.py b/pyrit/score/true_false/true_false_inverter_scorer.py index a8dc3f489..85cdd7374 100644 --- a/pyrit/score/true_false/true_false_inverter_scorer.py +++ b/pyrit/score/true_false/true_false_inverter_scorer.py @@ -4,6 +4,7 @@ import uuid from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_scorer import TrueFalseScorer @@ -30,9 +31,14 @@ def __init__(self, *, scorer: TrueFalseScorer, validator: Optional[ScorerPromptV super().__init__(validator=ScorerPromptValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index 6e5fdb483..2d50d780e 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -49,9 +50,14 @@ def __init__( self, validator=validator or self._default_validator, score_aggregator=score_aggregator ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """ + Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/tests/unit/identifiers/test_converter_identifier.py b/tests/unit/identifiers/test_converter_identifier.py new file mode 100644 index 000000000..1fef3cb00 --- /dev/null +++ b/tests/unit/identifiers/test_converter_identifier.py @@ -0,0 +1,222 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tests for ConverterIdentifier-specific functionality. + +Note: Base Identifier functionality (hash computation, to_dict/from_dict basics, +frozen/hashable properties) is tested via ScorerIdentifier in test_scorer_identifier.py. +These tests focus on converter-specific fields and behaviors. +""" + +from pyrit.identifiers import ConverterIdentifier + + +class TestConverterIdentifierSpecificFields: + """Test ConverterIdentifier-specific fields: supported_input_types, supported_output_types, converter_specific_params.""" + + def test_supported_input_output_types_stored_as_tuples(self): + """Test that supported_input_types and supported_output_types are stored correctly.""" + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text", "image_path"), + supported_output_types=("text",), + ) + + assert identifier.supported_input_types == ("text", "image_path") + assert identifier.supported_output_types == ("text",) + + def test_converter_specific_params_stored(self): + """Test that converter_specific_params are stored correctly.""" + identifier = ConverterIdentifier( + class_name="CaesarConverter", + class_module="pyrit.prompt_converter.caesar_converter", + class_description="A Caesar cipher converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + converter_specific_params={"shift": 3, "preserve_case": True}, + ) + + assert identifier.converter_specific_params["shift"] == 3 + assert identifier.converter_specific_params["preserve_case"] is True + + def test_sub_identifier_with_nested_converter(self): + """Test that sub_identifier can hold nested ConverterIdentifier.""" + sub_converter = ConverterIdentifier( + class_name="SubConverter", + class_module="pyrit.prompt_converter.sub_converter", + class_description="A sub converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + ) + + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + sub_identifier=[sub_converter], + ) + + assert len(identifier.sub_identifier) == 1 + assert identifier.sub_identifier[0].class_name == "SubConverter" + + +class TestConverterIdentifierHashDifferences: + """Test that converter-specific fields affect hash computation.""" + + def test_hash_different_for_different_input_types(self): + """Test that different input types produce different hashes.""" + base_args = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_output_types": ("text",), + } + + identifier1 = ConverterIdentifier(supported_input_types=("text",), **base_args) + identifier2 = ConverterIdentifier(supported_input_types=("image_path",), **base_args) + + assert identifier1.hash != identifier2.hash + + def test_hash_different_for_different_output_types(self): + """Test that different output types produce different hashes.""" + base_args = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_input_types": ("text",), + } + + identifier1 = ConverterIdentifier(supported_output_types=("text",), **base_args) + identifier2 = ConverterIdentifier(supported_output_types=("image_path",), **base_args) + + assert identifier1.hash != identifier2.hash + + def test_hash_different_for_different_converter_specific_params(self): + """Test that different converter_specific_params produce different hashes.""" + base_args = { + "class_name": "CaesarConverter", + "class_module": "pyrit.prompt_converter.caesar_converter", + "class_description": "A Caesar cipher converter", + "identifier_type": "instance", + "supported_input_types": ("text",), + "supported_output_types": ("text",), + } + + identifier1 = ConverterIdentifier(converter_specific_params={"shift": 3}, **base_args) + identifier2 = ConverterIdentifier(converter_specific_params={"shift": 5}, **base_args) + + assert identifier1.hash != identifier2.hash + + +class TestConverterIdentifierToDict: + """Test to_dict includes converter-specific fields.""" + + def test_to_dict_includes_supported_types(self): + """Test that supported_input_types and supported_output_types are in to_dict.""" + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text", "image_path"), + supported_output_types=("text",), + ) + + result = identifier.to_dict() + + # Tuples remain as tuples in to_dict + assert result["supported_input_types"] == ("text", "image_path") + assert result["supported_output_types"] == ("text",) + + def test_to_dict_includes_converter_specific_params(self): + """Test that converter_specific_params are included in to_dict.""" + identifier = ConverterIdentifier( + class_name="CaesarConverter", + class_module="pyrit.prompt_converter.caesar_converter", + class_description="A Caesar cipher converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + converter_specific_params={"shift": 13, "preserve_case": True}, + ) + + result = identifier.to_dict() + + assert result["converter_specific_params"] == {"shift": 13, "preserve_case": True} + + +class TestConverterIdentifierFromDict: + """Test from_dict handles converter-specific fields.""" + + def test_from_dict_converts_lists_to_tuples(self): + """Test that from_dict converts supported_*_types lists to tuples.""" + data = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_input_types": ["text", "image_path"], # List from JSON + "supported_output_types": ["text"], # List from JSON + } + + identifier = ConverterIdentifier.from_dict(data) + + # Lists should be converted to tuples + assert identifier.supported_input_types == ("text", "image_path") + assert identifier.supported_output_types == ("text",) + assert isinstance(identifier.supported_input_types, tuple) + assert isinstance(identifier.supported_output_types, tuple) + + def test_from_dict_provides_defaults_for_missing_types(self): + """Test that from_dict provides defaults for missing supported_*_types fields.""" + data = { + "class_name": "LegacyConverter", + "class_module": "pyrit.prompt_converter.legacy", + "class_description": "A legacy converter", + "identifier_type": "instance", + # Missing supported_input_types and supported_output_types + } + + identifier = ConverterIdentifier.from_dict(data) + + # Should provide empty tuples as defaults + assert identifier.supported_input_types == () + assert identifier.supported_output_types == () + + def test_from_dict_with_nested_sub_identifiers(self): + """Test from_dict with nested sub_identifier list creates ConverterIdentifier objects.""" + data = { + "class_name": "PipelineConverter", + "class_module": "pyrit.prompt_converter.pipeline_converter", + "class_description": "A pipeline converter", + "identifier_type": "instance", + "supported_input_types": ["text"], + "supported_output_types": ["text"], + "sub_identifier": [ + { + "class_name": "SubConverter", + "class_module": "pyrit.prompt_converter.sub", + "class_description": "Sub converter", + "identifier_type": "instance", + "supported_input_types": ["text"], + "supported_output_types": ["image_path"], + }, + ], + } + + identifier = ConverterIdentifier.from_dict(data) + + assert len(identifier.sub_identifier) == 1 + assert isinstance(identifier.sub_identifier[0], ConverterIdentifier) + assert identifier.sub_identifier[0].supported_output_types == ("image_path",) diff --git a/tests/unit/memory/test_azure_sql_memory.py b/tests/unit/memory/test_azure_sql_memory.py index 0f6033edb..0460febd6 100644 --- a/tests/unit/memory/test_azure_sql_memory.py +++ b/tests/unit/memory/test_azure_sql_memory.py @@ -204,7 +204,7 @@ def test_get_memories_with_json_properties(memory_interface: AzureSQLMemory): converter_identifiers = retrieved_entry.converter_identifiers assert len(converter_identifiers) == 1 - assert converter_identifiers[0]["__type__"] == "Base64Converter" + assert converter_identifiers[0].class_name == "Base64Converter" prompt_target = retrieved_entry.prompt_target_identifier assert prompt_target["__type__"] == "TextTarget" diff --git a/tests/unit/memory/test_sqlite_memory.py b/tests/unit/memory/test_sqlite_memory.py index a18e84e7e..f52f04198 100644 --- a/tests/unit/memory/test_sqlite_memory.py +++ b/tests/unit/memory/test_sqlite_memory.py @@ -373,7 +373,7 @@ def test_get_memories_with_json_properties(sqlite_instance): converter_identifiers = retrieved_entry.converter_identifiers assert len(converter_identifiers) == 1 - assert converter_identifiers[0]["__type__"] == "Base64Converter" + assert converter_identifiers[0].class_name == "Base64Converter" prompt_target = retrieved_entry.prompt_target_identifier assert prompt_target["__type__"] == "TextTarget" diff --git a/tests/unit/models/test_message_piece.py b/tests/unit/models/test_message_piece.py index c4aa7d81e..fced766ab 100644 --- a/tests/unit/models/test_message_piece.py +++ b/tests/unit/models/test_message_piece.py @@ -65,8 +65,8 @@ def test_converters_serialize(): converter = entry.converter_identifiers[0] - assert converter["__type__"] == "Base64Converter" - assert converter["__module__"] == "pyrit.prompt_converter.base64_converter" + assert converter.class_name == "Base64Converter" + assert converter.class_module == "pyrit.prompt_converter.base64_converter" def test_prompt_targets_serialize(patch_central_database): @@ -744,7 +744,7 @@ def test_message_piece_to_dict(): assert result["labels"] == entry.labels assert result["targeted_harm_categories"] == entry.targeted_harm_categories assert result["prompt_metadata"] == entry.prompt_metadata - assert result["converter_identifiers"] == entry.converter_identifiers + assert result["converter_identifiers"] == [conv.to_dict() for conv in entry.converter_identifiers] assert result["prompt_target_identifier"] == entry.prompt_target_identifier assert result["attack_identifier"] == entry.attack_identifier assert result["scorer_identifier"] == entry.scorer_identifier.to_dict() diff --git a/tests/unit/registry/test_scorer_registry.py b/tests/unit/registry/test_scorer_registry.py index 5a9d8ce7a..36421ab00 100644 --- a/tests/unit/registry/test_scorer_registry.py +++ b/tests/unit/registry/test_scorer_registry.py @@ -28,9 +28,13 @@ class MockTrueFalseScorer(TrueFalseScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -48,9 +52,13 @@ class MockFloatScaleScorer(FloatScaleScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -68,9 +76,13 @@ class MockGenericScorer(Scorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] diff --git a/tests/unit/score/test_conversation_history_scorer.py b/tests/unit/score/test_conversation_history_scorer.py index 29b956414..0cdbd6cae 100644 --- a/tests/unit/score/test_conversation_history_scorer.py +++ b/tests/unit/score/test_conversation_history_scorer.py @@ -38,7 +38,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -51,7 +51,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -64,7 +64,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index fcd441aa2..5cbb53940 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -9,6 +9,7 @@ import pytest from pyrit.exceptions import InvalidJsonException, remove_markdown_json +from pyrit.identifiers import ScorerIdentifier from pyrit.memory import CentralMemory from pyrit.models import Message, MessagePiece, Score from pyrit.prompt_target import PromptChatTarget @@ -60,9 +61,9 @@ class MockScorer(TrueFalseScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -116,9 +117,9 @@ def __init__(self, *, validator: ScorerPromptValidator): self.scored_piece_ids: list[str] = [] super().__init__(validator=validator) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: # Track which pieces get scored @@ -1118,9 +1119,9 @@ def __init__(self): self.scored_piece_ids = [] super().__init__(validator=validator) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this test scorer.""" - self._set_identifier() + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None @@ -1305,8 +1306,8 @@ class TestTrueFalseScorer(TrueFalseScorer): def __init__(self, validator): super().__init__(validator=validator) - def _build_identifier(self) -> None: - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_true_false_composite_scorer.py b/tests/unit/score/test_true_false_composite_scorer.py index ca0d4472d..fc40cd1ce 100644 --- a/tests/unit/score/test_true_false_composite_scorer.py +++ b/tests/unit/score/test_true_false_composite_scorer.py @@ -41,9 +41,13 @@ def __init__(self, score_value: bool, score_rationale: str, aggregator=None): # Call super().__init__() to properly initialize the scorer including _identifier super().__init__(validator=MagicMock()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -156,8 +160,8 @@ class InvalidScorer(FloatScaleScorer): def __init__(self): self._validator = MagicMock() - def _build_identifier(self) -> None: - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_video_scorer.py b/tests/unit/score/test_video_scorer.py index acab4b162..27de6693a 100644 --- a/tests/unit/score/test_video_scorer.py +++ b/tests/unit/score/test_video_scorer.py @@ -9,6 +9,7 @@ import numpy as np import pytest +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.float_scale.video_float_scale_scorer import VideoFloatScaleScorer @@ -68,9 +69,13 @@ def __init__(self, return_value: bool = True): validator = ScorerPromptValidator(supported_data_types=["image_path"]) super().__init__(validator=validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -96,9 +101,13 @@ def __init__(self, return_value: float = 0.8): validator = ScorerPromptValidator(supported_data_types=["image_path"]) super().__init__(validator=validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [