From 17328ef6c450f164d73728759ee24efb1b537996 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 27 Jan 2026 15:54:21 -0800 Subject: [PATCH 1/5] converter identifier --- pyrit/identifiers/__init__.py | 2 + pyrit/identifiers/converter_identifier.py | 71 ++++++++++++++++++++ pyrit/identifiers/identifier.py | 2 +- pyrit/models/message_piece.py | 23 +++++-- pyrit/prompt_converter/prompt_converter.py | 78 +++++++++++++++++++--- 5 files changed, 160 insertions(+), 16 deletions(-) create mode 100644 pyrit/identifiers/converter_identifier.py diff --git a/pyrit/identifiers/__init__.py b/pyrit/identifiers/__init__.py index 8ca875ca3..a6e0d759b 100644 --- a/pyrit/identifiers/__init__.py +++ b/pyrit/identifiers/__init__.py @@ -7,6 +7,7 @@ class_name_to_snake_case, snake_case_to_class_name, ) +from pyrit.identifiers.converter_identifier import ConverterIdentifier from pyrit.identifiers.identifiable import Identifiable, IdentifierT, LegacyIdentifiable from pyrit.identifiers.identifier import ( Identifier, @@ -16,6 +17,7 @@ __all__ = [ "class_name_to_snake_case", + "ConverterIdentifier", "Identifiable", "Identifier", "IdentifierT", diff --git a/pyrit/identifiers/converter_identifier.py b/pyrit/identifiers/converter_identifier.py new file mode 100644 index 000000000..8ab41032e --- /dev/null +++ b/pyrit/identifiers/converter_identifier.py @@ -0,0 +1,71 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple, Type, cast + +from pyrit.identifiers.identifier import Identifier + + +@dataclass(frozen=True) +class ConverterIdentifier(Identifier): + """ + Identifier for PromptConverter instances. + + This frozen dataclass extends Identifier with converter-specific fields. + It provides a structured way to identify and track converters used in + prompt transformations. + """ + + supported_input_types: Tuple[str, ...] = field(kw_only=True) + """The input data types supported by this converter (e.g., ('text',), ('image', 'text')).""" + + supported_output_types: Tuple[str, ...] = field(kw_only=True) + """The output data types produced by this converter.""" + + sub_identifier: Optional[List["ConverterIdentifier"]] = None + """List of sub-converter identifiers for composite converters like ConverterPipeline.""" + + target_info: Optional[Dict[str, Any]] = None + """Information about the prompt target used by the converter (for LLM-based converters).""" + + converter_specific_params: Optional[Dict[str, Any]] = None + """Additional converter-specific parameters.""" + + @classmethod + def from_dict(cls: Type["ConverterIdentifier"], data: dict[str, Any]) -> "ConverterIdentifier": + """ + Create a ConverterIdentifier from a dictionary (e.g., retrieved from database). + + Extends the base Identifier.from_dict() to recursively reconstruct + nested ConverterIdentifier objects in sub_identifier. + + Args: + data: The dictionary representation. + + Returns: + ConverterIdentifier: A new ConverterIdentifier instance. + """ + # Create a mutable copy + data = dict(data) + + # Recursively reconstruct sub_identifier if present + if "sub_identifier" in data and data["sub_identifier"] is not None: + data["sub_identifier"] = [ + ConverterIdentifier.from_dict(sub) if isinstance(sub, dict) else sub for sub in data["sub_identifier"] + ] + + # Convert supported_input_types and supported_output_types from list to tuple if needed + if "supported_input_types" in data and data["supported_input_types"] is not None: + if isinstance(data["supported_input_types"], list): + data["supported_input_types"] = tuple(data["supported_input_types"]) + + if "supported_output_types" in data and data["supported_output_types"] is not None: + if isinstance(data["supported_output_types"], list): + data["supported_output_types"] = tuple(data["supported_output_types"]) + + # Delegate to parent class for standard processing + result = Identifier.from_dict.__func__(cls, data) # type: ignore[attr-defined] + return cast(ConverterIdentifier, result) diff --git a/pyrit/identifiers/identifier.py b/pyrit/identifiers/identifier.py index 8e5265d4b..3d698e717 100644 --- a/pyrit/identifiers/identifier.py +++ b/pyrit/identifiers/identifier.py @@ -64,7 +64,7 @@ def _compute_hash(self) -> str: """ Compute a stable SHA256 hash from storable identifier fields. - Fields marked with metadata={"exclude_from_storage": True}, 'hash', and 'name' + Fields marked with metadata={"exclude_from_storage": True}, 'hash', and 'unique_name' are excluded from the hash computation. Returns: diff --git a/pyrit/models/message_piece.py b/pyrit/models/message_piece.py index d0dc3e6df..af16fe612 100644 --- a/pyrit/models/message_piece.py +++ b/pyrit/models/message_piece.py @@ -9,7 +9,7 @@ from uuid import uuid4 from pyrit.common.deprecation import print_deprecation_message -from pyrit.identifiers import ScorerIdentifier +from pyrit.identifiers import ConverterIdentifier, ScorerIdentifier from pyrit.models.literals import ChatMessageRole, PromptDataType, PromptResponseError from pyrit.models.score import Score @@ -38,7 +38,7 @@ def __init__( sequence: int = -1, labels: Optional[Dict[str, str]] = None, prompt_metadata: Optional[Dict[str, Union[str, int]]] = None, - converter_identifiers: Optional[List[Dict[str, str]]] = None, + converter_identifiers: Optional[List[Union[ConverterIdentifier, Dict[str, str]]]] = None, prompt_target_identifier: Optional[Dict[str, str]] = None, attack_identifier: Optional[Dict[str, str]] = None, scorer_identifier: Optional[Union[ScorerIdentifier, Dict[str, str]]] = None, @@ -69,7 +69,8 @@ def __init__( Because memory is how components talk with each other, this can be component specific. e.g. the URI from a file uploaded to a blob store, or a document type you want to upload. Defaults to None. - converter_identifiers: The converter identifiers for the prompt. Defaults to None. + converter_identifiers: The converter identifiers for the prompt. Can be ConverterIdentifier + objects or dicts (deprecated, will be removed in 0.14.0). Defaults to None. prompt_target_identifier: The target identifier for the prompt. Defaults to None. attack_identifier: The attack identifier for the prompt. Defaults to None. scorer_identifier: The scorer identifier for the prompt. Can be a ScorerIdentifier or a @@ -106,7 +107,19 @@ def __init__( self.labels = labels or {} self.prompt_metadata = prompt_metadata or {} - self.converter_identifiers = converter_identifiers if converter_identifiers else [] + # Handle converter_identifiers: convert dicts to ConverterIdentifier with deprecation warning + self.converter_identifiers: List[ConverterIdentifier] = [] + if converter_identifiers: + for conv_id in converter_identifiers: + if isinstance(conv_id, dict): + print_deprecation_message( + old_item="dict for converter_identifiers", + new_item="ConverterIdentifier", + removed_in="0.14.0", + ) + self.converter_identifiers.append(ConverterIdentifier.from_dict(conv_id)) + else: + self.converter_identifiers.append(conv_id) self.prompt_target_identifier = prompt_target_identifier or {} self.attack_identifier = attack_identifier or {} @@ -278,7 +291,7 @@ def to_dict(self) -> dict[str, object]: "labels": self.labels, "targeted_harm_categories": self.targeted_harm_categories if self.targeted_harm_categories else None, "prompt_metadata": self.prompt_metadata, - "converter_identifiers": self.converter_identifiers, + "converter_identifiers": [conv.to_dict() for conv in self.converter_identifiers], "prompt_target_identifier": self.prompt_target_identifier, "attack_identifier": self.attack_identifier, "scorer_identifier": self.scorer_identifier.to_dict() if self.scorer_identifier else None, diff --git a/pyrit/prompt_converter/prompt_converter.py b/pyrit/prompt_converter/prompt_converter.py index d83f9a043..cfa2d34b1 100644 --- a/pyrit/prompt_converter/prompt_converter.py +++ b/pyrit/prompt_converter/prompt_converter.py @@ -6,10 +6,10 @@ import inspect import re from dataclasses import dataclass -from typing import get_args +from typing import Any, Dict, List, Optional, Sequence, get_args from pyrit import prompt_converter -from pyrit.identifiers import LegacyIdentifiable +from pyrit.identifiers import ConverterIdentifier, Identifiable from pyrit.models import PromptDataType @@ -32,7 +32,7 @@ def __str__(self) -> str: return f"{self.output_type}: {self.output_text}" -class PromptConverter(LegacyIdentifiable): +class PromptConverter(Identifiable[ConverterIdentifier]): """ Base class for converters that transform prompts into a different representation or format. @@ -48,6 +48,8 @@ class PromptConverter(LegacyIdentifiable): #: Tuple of output modalities supported by this converter. Subclasses must override this. SUPPORTED_OUTPUT_TYPES: tuple[PromptDataType, ...] = () + _identifier: Optional[ConverterIdentifier] = None + def __init_subclass__(cls, **kwargs: object) -> None: """ Validate that concrete subclasses define required class attributes. @@ -163,17 +165,73 @@ async def _replace_text_match(self, match: str) -> ConverterResult: result = await self.convert_async(prompt=match, input_type="text") return result - def get_identifier(self) -> dict[str, str]: + def get_identifier(self) -> ConverterIdentifier: """ - Return an identifier dictionary for the converter. + Get the converter identifier. Built lazily on first access. Returns: - dict: The identifier dictionary. + ConverterIdentifier: The identifier containing all configuration parameters. + """ + if self._identifier is None: + self._build_identifier() + assert self._identifier is not None, "_build_identifier must set _identifier" + return self._identifier + + def _build_identifier(self) -> None: + """ + Build the identifier for this converter. + + Subclasses can override this method to add converter-specific parameters + by calling _set_identifier with additional arguments. + + The default implementation calls _set_identifier with no extra parameters. + """ + self._set_identifier() + + def _set_identifier( + self, + *, + sub_converters: Optional[Sequence["PromptConverter"]] = None, + prompt_target: Optional[Any] = None, + converter_specific_params: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Construct the converter identifier. + + Args: + sub_converters: List of sub-converters for composite converters + (e.g., ConverterPipeline). Defaults to None. + prompt_target: The prompt target used by this converter (for LLM-based converters). + Defaults to None. + converter_specific_params: Additional converter-specific parameters. + Defaults to None. """ - public_attributes = {} - public_attributes["__type__"] = self.__class__.__name__ - public_attributes["__module__"] = self.__class__.__module__ - return public_attributes + # Build sub_identifier from sub_converters + sub_identifier: Optional[List[ConverterIdentifier]] = None + if sub_converters: + sub_identifier = [converter.get_identifier() for converter in sub_converters] + + # Extract target_info from prompt_target + target_info: Optional[Dict[str, Any]] = None + if prompt_target: + target_id = prompt_target.get_identifier() + # Extract standard fields for converter identification + target_info = {} + for key in ["__type__", "model_name", "temperature", "top_p"]: + if key in target_id: + target_info[key] = target_id[key] + + self._identifier = ConverterIdentifier( + class_name=self.__class__.__name__, + class_module=self.__class__.__module__, + class_description=self.__class__.__doc__ or "", + identifier_type="instance", + supported_input_types=self.SUPPORTED_INPUT_TYPES, + supported_output_types=self.SUPPORTED_OUTPUT_TYPES, + sub_identifier=sub_identifier, + target_info=target_info, + converter_specific_params=converter_specific_params, + ) @property def supported_input_types(self) -> list[PromptDataType]: From c65d3322c55d7eb377752a7a0113f5758d2657dc Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 27 Jan 2026 16:19:45 -0800 Subject: [PATCH 2/5] fixing tests --- doc/api.rst | 1 + pyrit/exceptions/exception_context.py | 20 ++++++++++++++----- .../attack/multi_turn/chunked_request.py | 2 +- pyrit/executor/attack/multi_turn/crescendo.py | 4 ++-- .../attack/multi_turn/multi_prompt_sending.py | 2 +- .../executor/attack/multi_turn/red_teaming.py | 2 +- .../attack/multi_turn/tree_of_attacks.py | 2 +- .../attack/single_turn/prompt_sending.py | 2 +- pyrit/identifiers/converter_identifier.py | 6 ++++++ pyrit/memory/memory_models.py | 2 +- tests/unit/memory/test_azure_sql_memory.py | 2 +- tests/unit/memory/test_sqlite_memory.py | 2 +- tests/unit/models/test_message_piece.py | 6 +++--- 13 files changed, 35 insertions(+), 18 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 89738196c..8fbec2512 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -272,6 +272,7 @@ API Reference :toctree: _autosummary/ class_name_to_snake_case + ConverterIdentifier Identifiable Identifier IdentifierT diff --git a/pyrit/exceptions/exception_context.py b/pyrit/exceptions/exception_context.py index 11515a413..bce5b9238 100644 --- a/pyrit/exceptions/exception_context.py +++ b/pyrit/exceptions/exception_context.py @@ -13,7 +13,9 @@ from contextvars import ContextVar from dataclasses import dataclass, field from enum import Enum -from typing import Any, Dict, Optional +from typing import Any, Dict, Optional, Union + +from pyrit.identifiers import Identifier class ComponentRole(Enum): @@ -191,7 +193,7 @@ def execution_context( component_role: ComponentRole, attack_strategy_name: Optional[str] = None, attack_identifier: Optional[Dict[str, Any]] = None, - component_identifier: Optional[Dict[str, Any]] = None, + component_identifier: Optional[Union[Identifier, Dict[str, Any]]] = None, objective_target_conversation_id: Optional[str] = None, objective: Optional[str] = None, ) -> ExecutionContextManager: @@ -203,6 +205,7 @@ def execution_context( attack_strategy_name: The name of the attack strategy class. attack_identifier: The identifier from attack.get_identifier(). component_identifier: The identifier from component.get_identifier(). + Can be an Identifier object or a dict (legacy format). objective_target_conversation_id: The objective target conversation ID if available. objective: The attack objective if available. @@ -212,15 +215,22 @@ def execution_context( # Extract endpoint and component_name from component_identifier if available endpoint = None component_name = None + component_id_dict: Optional[Dict[str, Any]] = None if component_identifier: - endpoint = component_identifier.get("endpoint") - component_name = component_identifier.get("__type__") + if isinstance(component_identifier, Identifier): + endpoint = getattr(component_identifier, "endpoint", None) + component_name = component_identifier.class_name + component_id_dict = component_identifier.to_dict() + else: + endpoint = component_identifier.get("endpoint") + component_name = component_identifier.get("__type__") + component_id_dict = component_identifier context = ExecutionContext( component_role=component_role, attack_strategy_name=attack_strategy_name, attack_identifier=attack_identifier, - component_identifier=component_identifier, + component_identifier=component_id_dict, objective_target_conversation_id=objective_target_conversation_id, endpoint=endpoint, component_name=component_name, diff --git a/pyrit/executor/attack/multi_turn/chunked_request.py b/pyrit/executor/attack/multi_turn/chunked_request.py index 7749e11c0..feabb9821 100644 --- a/pyrit/executor/attack/multi_turn/chunked_request.py +++ b/pyrit/executor/attack/multi_turn/chunked_request.py @@ -362,7 +362,7 @@ async def _score_combined_value_async( component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective=objective, ): scores = await self._objective_scorer.score_text_async(text=combined_value, objective=objective) diff --git a/pyrit/executor/attack/multi_turn/crescendo.py b/pyrit/executor/attack/multi_turn/crescendo.py index a6a7d2049..7a274053c 100644 --- a/pyrit/executor/attack/multi_turn/crescendo.py +++ b/pyrit/executor/attack/multi_turn/crescendo.py @@ -636,7 +636,7 @@ async def _check_refusal_async(self, context: CrescendoAttackContext, objective: component_role=ComponentRole.REFUSAL_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._refusal_scorer.get_identifier().to_dict(), + component_identifier=self._refusal_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): @@ -666,7 +666,7 @@ async def _score_response_async(self, *, context: CrescendoAttackContext) -> Sco component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): diff --git a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py index 0b3464aaa..c451af7c9 100644 --- a/pyrit/executor/attack/multi_turn/multi_prompt_sending.py +++ b/pyrit/executor/attack/multi_turn/multi_prompt_sending.py @@ -373,7 +373,7 @@ async def _evaluate_response_async(self, *, response: Message, objective: str) - component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict() if self._objective_scorer else None, + component_identifier=self._objective_scorer.get_identifier() if self._objective_scorer else None, objective=objective, ): scoring_results = await Scorer.score_response_async( diff --git a/pyrit/executor/attack/multi_turn/red_teaming.py b/pyrit/executor/attack/multi_turn/red_teaming.py index 619446bcc..33b2c75d7 100644 --- a/pyrit/executor/attack/multi_turn/red_teaming.py +++ b/pyrit/executor/attack/multi_turn/red_teaming.py @@ -573,7 +573,7 @@ async def _score_response_async(self, *, context: MultiTurnAttackContext[Any]) - component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=context.session.conversation_id, objective=context.objective, ): diff --git a/pyrit/executor/attack/multi_turn/tree_of_attacks.py b/pyrit/executor/attack/multi_turn/tree_of_attacks.py index d71875333..cfad81d3a 100644 --- a/pyrit/executor/attack/multi_turn/tree_of_attacks.py +++ b/pyrit/executor/attack/multi_turn/tree_of_attacks.py @@ -639,7 +639,7 @@ async def _score_response_async(self, *, response: Message, objective: str) -> N component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self._attack_strategy_name, attack_identifier=self._attack_id, - component_identifier=self._objective_scorer.get_identifier().to_dict(), + component_identifier=self._objective_scorer.get_identifier(), objective_target_conversation_id=self.objective_target_conversation_id, objective=objective, ): diff --git a/pyrit/executor/attack/single_turn/prompt_sending.py b/pyrit/executor/attack/single_turn/prompt_sending.py index 299aba643..3b3f62350 100644 --- a/pyrit/executor/attack/single_turn/prompt_sending.py +++ b/pyrit/executor/attack/single_turn/prompt_sending.py @@ -355,7 +355,7 @@ async def _evaluate_response_async( component_role=ComponentRole.OBJECTIVE_SCORER, attack_strategy_name=self.__class__.__name__, attack_identifier=self.get_identifier(), - component_identifier=self._objective_scorer.get_identifier().to_dict() if self._objective_scorer else None, + component_identifier=self._objective_scorer.get_identifier() if self._objective_scorer else None, objective=objective, ): scoring_results = await Scorer.score_response_async( diff --git a/pyrit/identifiers/converter_identifier.py b/pyrit/identifiers/converter_identifier.py index 8ab41032e..777672a93 100644 --- a/pyrit/identifiers/converter_identifier.py +++ b/pyrit/identifiers/converter_identifier.py @@ -61,10 +61,16 @@ def from_dict(cls: Type["ConverterIdentifier"], data: dict[str, Any]) -> "Conver if "supported_input_types" in data and data["supported_input_types"] is not None: if isinstance(data["supported_input_types"], list): data["supported_input_types"] = tuple(data["supported_input_types"]) + else: + # Provide default for legacy dicts that don't have this field + data["supported_input_types"] = () if "supported_output_types" in data and data["supported_output_types"] is not None: if isinstance(data["supported_output_types"], list): data["supported_output_types"] = tuple(data["supported_output_types"]) + else: + # Provide default for legacy dicts that don't have this field + data["supported_output_types"] = () # Delegate to parent class for standard processing result = Identifier.from_dict.__func__(cls, data) # type: ignore[attr-defined] diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 1401c2e24..c7139b576 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -207,7 +207,7 @@ def __init__(self, *, entry: MessagePiece): self.labels = entry.labels self.prompt_metadata = entry.prompt_metadata self.targeted_harm_categories = entry.targeted_harm_categories - self.converter_identifiers = entry.converter_identifiers + self.converter_identifiers = [conv.to_dict() for conv in entry.converter_identifiers] self.prompt_target_identifier = entry.prompt_target_identifier self.attack_identifier = entry.attack_identifier diff --git a/tests/unit/memory/test_azure_sql_memory.py b/tests/unit/memory/test_azure_sql_memory.py index 0f6033edb..0460febd6 100644 --- a/tests/unit/memory/test_azure_sql_memory.py +++ b/tests/unit/memory/test_azure_sql_memory.py @@ -204,7 +204,7 @@ def test_get_memories_with_json_properties(memory_interface: AzureSQLMemory): converter_identifiers = retrieved_entry.converter_identifiers assert len(converter_identifiers) == 1 - assert converter_identifiers[0]["__type__"] == "Base64Converter" + assert converter_identifiers[0].class_name == "Base64Converter" prompt_target = retrieved_entry.prompt_target_identifier assert prompt_target["__type__"] == "TextTarget" diff --git a/tests/unit/memory/test_sqlite_memory.py b/tests/unit/memory/test_sqlite_memory.py index a18e84e7e..f52f04198 100644 --- a/tests/unit/memory/test_sqlite_memory.py +++ b/tests/unit/memory/test_sqlite_memory.py @@ -373,7 +373,7 @@ def test_get_memories_with_json_properties(sqlite_instance): converter_identifiers = retrieved_entry.converter_identifiers assert len(converter_identifiers) == 1 - assert converter_identifiers[0]["__type__"] == "Base64Converter" + assert converter_identifiers[0].class_name == "Base64Converter" prompt_target = retrieved_entry.prompt_target_identifier assert prompt_target["__type__"] == "TextTarget" diff --git a/tests/unit/models/test_message_piece.py b/tests/unit/models/test_message_piece.py index c4aa7d81e..fced766ab 100644 --- a/tests/unit/models/test_message_piece.py +++ b/tests/unit/models/test_message_piece.py @@ -65,8 +65,8 @@ def test_converters_serialize(): converter = entry.converter_identifiers[0] - assert converter["__type__"] == "Base64Converter" - assert converter["__module__"] == "pyrit.prompt_converter.base64_converter" + assert converter.class_name == "Base64Converter" + assert converter.class_module == "pyrit.prompt_converter.base64_converter" def test_prompt_targets_serialize(patch_central_database): @@ -744,7 +744,7 @@ def test_message_piece_to_dict(): assert result["labels"] == entry.labels assert result["targeted_harm_categories"] == entry.targeted_harm_categories assert result["prompt_metadata"] == entry.prompt_metadata - assert result["converter_identifiers"] == entry.converter_identifiers + assert result["converter_identifiers"] == [conv.to_dict() for conv in entry.converter_identifiers] assert result["prompt_target_identifier"] == entry.prompt_target_identifier assert result["attack_identifier"] == entry.attack_identifier assert result["scorer_identifier"] == entry.scorer_identifier.to_dict() From 1aba154259427116c3b1ea01a9de1fd02b92fa19 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 27 Jan 2026 18:31:56 -0800 Subject: [PATCH 3/5] implementing converterIdentifier --- pyrit/identifiers/identifiable.py | 27 +++++++------ pyrit/memory/memory_models.py | 11 +++-- .../add_image_text_converter.py | 18 +++++++++ .../add_image_to_video_converter.py | 15 +++++++ .../add_text_image_converter.py | 20 ++++++++++ pyrit/prompt_converter/ascii_art_converter.py | 17 +++++++- pyrit/prompt_converter/atbash_converter.py | 13 ++++++ .../audio_frequency_converter.py | 14 +++++++ .../azure_speech_audio_to_text_converter.py | 13 ++++++ .../azure_speech_text_to_audio_converter.py | 15 +++++++ pyrit/prompt_converter/base64_converter.py | 13 ++++++ pyrit/prompt_converter/bin_ascii_converter.py | 11 +++++ pyrit/prompt_converter/binary_converter.py | 11 +++++ pyrit/prompt_converter/caesar_converter.py | 14 +++++++ .../charswap_attack_converter.py | 17 +++++++- .../codechameleon_converter.py | 15 +++++++ .../colloquial_wordswap_converter.py | 14 +++++++ pyrit/prompt_converter/diacritic_converter.py | 14 +++++++ .../first_letter_converter.py | 11 +++++ .../human_in_the_loop_converter.py | 9 +++++ .../image_compression_converter.py | 19 +++++++++ .../insert_punctuation_converter.py | 14 +++++++ pyrit/prompt_converter/leetspeak_converter.py | 24 +++++++++++ .../llm_generic_text_converter.py | 29 ++++++++++++++ .../math_obfuscation_converter.py | 14 +++++++ pyrit/prompt_converter/morse_converter.py | 13 ++++++ pyrit/prompt_converter/noise_converter.py | 17 ++++++++ pyrit/prompt_converter/pdf_converter.py | 27 +++++++++++++ .../prompt_converter/persuasion_converter.py | 15 +++++++ pyrit/prompt_converter/prompt_converter.py | 40 ++++++++----------- pyrit/prompt_converter/qr_code_converter.py | 16 ++++++++ .../random_capital_letters_converter.py | 13 ++++++ .../repeat_token_converter.py | 22 ++++++++-- .../search_replace_converter.py | 23 ++++++++--- .../selective_text_converter.py | 17 ++++++++ .../prompt_converter/string_join_converter.py | 17 +++++++- .../suffix_append_converter.py | 17 +++++++- .../template_segment_converter.py | 16 ++++++++ pyrit/prompt_converter/tense_converter.py | 15 +++++++ .../text_jailbreak_converter.py | 13 ++++++ .../ascii_smuggler_converter.py | 11 +++++ .../prompt_converter/token_smuggling/base.py | 13 ++++++ .../sneaky_bits_smuggler_converter.py | 12 ++++++ .../variation_selector_smuggler_converter.py | 12 ++++++ pyrit/prompt_converter/tone_converter.py | 15 +++++++ .../prompt_converter/translation_converter.py | 14 +++++++ .../transparency_attack_converter.py | 16 ++++++++ .../unicode_confusable_converter.py | 14 +++++++ .../unicode_replacement_converter.py | 11 +++++ .../prompt_converter/unicode_sub_converter.py | 13 ++++++ pyrit/prompt_converter/variation_converter.py | 11 +++++ .../prompt_converter/word_level_converter.py | 14 +++++++ pyrit/prompt_converter/zalgo_converter.py | 13 ++++++ pyrit/score/conversation_scorer.py | 11 +++-- .../azure_content_filter_scorer.py | 11 +++-- .../score/float_scale/insecure_code_scorer.py | 11 +++-- pyrit/score/float_scale/plagiarism_scorer.py | 11 +++-- .../self_ask_general_float_scale_scorer.py | 11 +++-- .../float_scale/self_ask_likert_scorer.py | 11 +++-- .../float_scale/self_ask_scale_scorer.py | 11 +++-- .../float_scale/video_float_scale_scorer.py | 11 +++-- pyrit/score/human/human_in_the_loop_gradio.py | 11 +++-- pyrit/score/scorer.py | 21 +++------- pyrit/score/true_false/decoding_scorer.py | 11 +++-- .../float_scale_threshold_scorer.py | 11 +++-- pyrit/score/true_false/gandalf_scorer.py | 11 +++-- pyrit/score/true_false/markdown_injection.py | 11 +++-- .../score/true_false/prompt_shield_scorer.py | 11 +++-- .../true_false/question_answer_scorer.py | 11 +++-- .../true_false/self_ask_category_scorer.py | 11 +++-- .../self_ask_general_true_false_scorer.py | 11 +++-- .../true_false/self_ask_refusal_scorer.py | 11 +++-- .../true_false/self_ask_true_false_scorer.py | 11 +++-- pyrit/score/true_false/substring_scorer.py | 11 +++-- .../true_false/true_false_composite_scorer.py | 11 +++-- .../true_false/true_false_inverter_scorer.py | 11 +++-- .../true_false/video_true_false_scorer.py | 11 +++-- tests/unit/registry/test_scorer_registry.py | 30 +++++++++----- tests/unit/score/test_scorer.py | 17 ++++---- .../score/test_true_false_composite_scorer.py | 14 ++++--- tests/unit/score/test_video_scorer.py | 21 +++++++--- 81 files changed, 1032 insertions(+), 166 deletions(-) diff --git a/pyrit/identifiers/identifiable.py b/pyrit/identifiers/identifiable.py index b588918d8..94108e6ea 100644 --- a/pyrit/identifiers/identifiable.py +++ b/pyrit/identifiers/identifiable.py @@ -4,7 +4,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from typing import Generic, TypeVar +from typing import Generic, Optional, TypeVar from pyrit.identifiers.identifier import Identifier @@ -37,29 +37,32 @@ class Identifiable(ABC, Generic[IdentifierT]): Generic over IdentifierT, allowing subclasses to specify their exact identifier type for strong typing support. - Subclasses must: - 1. Implement `_build_identifier()` to construct their specific identifier - 2. Implement `get_identifier()` to return the typed identifier (can use lazy building) + Subclasses must implement `_build_identifier()` to construct their specific identifier. + The `get_identifier()` method is provided and uses lazy building with caching. """ + _identifier: Optional[IdentifierT] = None + @abstractmethod - def _build_identifier(self) -> None: + def _build_identifier(self) -> IdentifierT: """ - Build the identifier for this object. + Build and return the identifier for this object. - Subclasses must implement this method to construct their specific identifier type - and store it in an instance variable (typically `_identifier`). + Subclasses must implement this method to construct their specific identifier type. + This method is called lazily on first access via `get_identifier()`. - This method is typically called lazily on first access via `get_identifier()`. + Returns: + IdentifierT: The constructed identifier for this component. """ raise NotImplementedError("Subclasses must implement _build_identifier") - @abstractmethod def get_identifier(self) -> IdentifierT: """ - Get the typed identifier for this object. + Get the typed identifier for this object. Built lazily on first access. Returns: IdentifierT: The identifier for this component. """ - ... + if self._identifier is None: + self._identifier = self._build_identifier() + return self._identifier diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index c7139b576..2cccbf056 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -31,7 +31,7 @@ from sqlalchemy.types import Uuid from pyrit.common.utils import to_sha256 -from pyrit.identifiers import ScorerIdentifier +from pyrit.identifiers import ConverterIdentifier, ScorerIdentifier from pyrit.models import ( AttackOutcome, AttackResult, @@ -164,7 +164,7 @@ class PromptMemoryEntry(Base): labels: Mapped[dict[str, str]] = mapped_column(JSON) prompt_metadata: Mapped[dict[str, Union[str, int]]] = mapped_column(JSON) targeted_harm_categories: Mapped[Optional[List[str]]] = mapped_column(JSON) - converter_identifiers: Mapped[Optional[List[dict[str, str]]]] = mapped_column(JSON) + converter_identifiers: Mapped[Optional[List[Dict[str, str]]]] = mapped_column(JSON) prompt_target_identifier: Mapped[dict[str, str]] = mapped_column(JSON) attack_identifier: Mapped[dict[str, str]] = mapped_column(JSON) response_error: Mapped[Literal["blocked", "none", "processing", "unknown"]] = mapped_column(String, nullable=True) @@ -230,6 +230,11 @@ def get_message_piece(self) -> MessagePiece: Returns: MessagePiece: The reconstructed message piece with all its data and scores. """ + converter_ids = ( + [ConverterIdentifier.from_dict(c) for c in self.converter_identifiers] + if self.converter_identifiers + else None + ) message_piece = MessagePiece( role=self.role, original_value=self.original_value, @@ -242,7 +247,7 @@ def get_message_piece(self) -> MessagePiece: labels=self.labels, prompt_metadata=self.prompt_metadata, targeted_harm_categories=self.targeted_harm_categories, - converter_identifiers=self.converter_identifiers, + converter_identifiers=converter_ids, prompt_target_identifier=self.prompt_target_identifier, attack_identifier=self.attack_identifier, original_value_data_type=self.original_value_data_type, diff --git a/pyrit/prompt_converter/add_image_text_converter.py b/pyrit/prompt_converter/add_image_text_converter.py index ca417c183..63429af2c 100644 --- a/pyrit/prompt_converter/add_image_text_converter.py +++ b/pyrit/prompt_converter/add_image_text_converter.py @@ -11,6 +11,7 @@ from PIL import Image, ImageDraw, ImageFont from PIL.ImageFont import FreeTypeFont +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -63,6 +64,23 @@ def __init__( self._x_pos = x_pos self._y_pos = y_pos + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with image and text parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "img_to_add_path": str(self._img_to_add), + "font_name": self._font_name, + "color": self._color, + "font_size": self._font_size, + "x_pos": self._x_pos, + "y_pos": self._y_pos, + }, + ) + def _load_font(self) -> FreeTypeFont: """ Load the font for a given font name and font size. diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index 77c0347f0..dc39e5350 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -9,6 +9,7 @@ import numpy as np from pyrit.common.path import DB_DATA_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -61,6 +62,20 @@ def __init__( self._img_resize_size = img_resize_size self._video_path = video_path + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with video converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "video_path": str(self._video_path), + "img_position": self._img_position, + "img_resize_size": self._img_resize_size, + } + ) + async def _add_image_to_video(self, image_path: str, output_path: str) -> str: """ Add an image to video. diff --git a/pyrit/prompt_converter/add_text_image_converter.py b/pyrit/prompt_converter/add_text_image_converter.py index fbf121cc4..9a62b1aee 100644 --- a/pyrit/prompt_converter/add_text_image_converter.py +++ b/pyrit/prompt_converter/add_text_image_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import base64 +import hashlib import logging import string import textwrap @@ -11,6 +12,7 @@ from PIL import Image, ImageDraw, ImageFont from PIL.ImageFont import FreeTypeFont +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -63,6 +65,24 @@ def __init__( self._x_pos = x_pos self._y_pos = y_pos + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with text and image parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + text_hash = hashlib.sha256(self._text_to_add.encode("utf-8")).hexdigest()[:16] + return self._set_identifier( + converter_specific_params={ + "text_to_add_hash": text_hash, + "font_name": self._font_name, + "color": self._color, + "font_size": self._font_size, + "x_pos": self._x_pos, + "y_pos": self._y_pos, + }, + ) + def _load_font(self) -> FreeTypeFont: """ Load the font for a given font name and font size. diff --git a/pyrit/prompt_converter/ascii_art_converter.py b/pyrit/prompt_converter/ascii_art_converter.py index be8fc29b2..fc9991880 100644 --- a/pyrit/prompt_converter/ascii_art_converter.py +++ b/pyrit/prompt_converter/ascii_art_converter.py @@ -3,6 +3,7 @@ from art import text2art +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -22,7 +23,19 @@ def __init__(self, font: str = "rand") -> None: Args: font (str): The font to use for ASCII art. Defaults to "rand" which selects a random font. """ - self.font_value = font + self._font = font + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with font parameter. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "font": self._font, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -41,4 +54,4 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - return ConverterResult(output_text=text2art(prompt, font=self.font_value), output_type="text") + return ConverterResult(output_text=text2art(prompt, font=self._font), output_type="text") diff --git a/pyrit/prompt_converter/atbash_converter.py b/pyrit/prompt_converter/atbash_converter.py index d8ee91ba8..4acc94205 100644 --- a/pyrit/prompt_converter/atbash_converter.py +++ b/pyrit/prompt_converter/atbash_converter.py @@ -5,6 +5,7 @@ import string from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -39,6 +40,18 @@ def __init__(self, *, append_description: bool = False) -> None: "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with Atbash cipher parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "append_description": self.append_description, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the Atbash cipher. diff --git a/pyrit/prompt_converter/audio_frequency_converter.py b/pyrit/prompt_converter/audio_frequency_converter.py index 33153a13d..f308d7503 100644 --- a/pyrit/prompt_converter/audio_frequency_converter.py +++ b/pyrit/prompt_converter/audio_frequency_converter.py @@ -8,6 +8,7 @@ import numpy as np from scipy.io import wavfile +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -42,6 +43,19 @@ def __init__( self._output_format = output_format self._shift_value = shift_value + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with audio frequency parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "output_format": self._output_format, + "shift_value": self._shift_value, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "audio_path") -> ConverterResult: """ Convert the given audio file by shifting its frequency. diff --git a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py index 77352e589..97fb929c3 100644 --- a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py +++ b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py @@ -10,6 +10,7 @@ from pyrit.auth.azure_auth import get_speech_config from pyrit.common import default_values +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -86,6 +87,18 @@ def __init__( # Create a flag to indicate when recognition is finished self.done = False + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with speech recognition parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "recognition_language": self._recognition_language, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "audio_path") -> ConverterResult: """ Convert the given audio file into its text representation. diff --git a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py index 87db4b71d..09defe733 100644 --- a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py +++ b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py @@ -9,6 +9,7 @@ from pyrit.auth.azure_auth import get_speech_config from pyrit.common import default_values +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -93,6 +94,20 @@ def __init__( self._synthesis_voice_name = synthesis_voice_name self._output_format = output_format + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with speech synthesis parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "synthesis_language": self._synthesis_language, + "synthesis_voice_name": self._synthesis_voice_name, + "output_format": self._output_format, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given text prompt into its audio representation. diff --git a/pyrit/prompt_converter/base64_converter.py b/pyrit/prompt_converter/base64_converter.py index 24105fe0d..f7b25a0b5 100644 --- a/pyrit/prompt_converter/base64_converter.py +++ b/pyrit/prompt_converter/base64_converter.py @@ -5,6 +5,7 @@ import binascii from typing import Literal +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -44,6 +45,18 @@ def __init__(self, *, encoding_func: EncodingFunc = "b64encode") -> None: """ self._encoding_func = encoding_func + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with encoding function. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "encoding_func": self._encoding_func, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to base64 encoding. diff --git a/pyrit/prompt_converter/bin_ascii_converter.py b/pyrit/prompt_converter/bin_ascii_converter.py index 84c71f291..3e55d4f42 100644 --- a/pyrit/prompt_converter/bin_ascii_converter.py +++ b/pyrit/prompt_converter/bin_ascii_converter.py @@ -4,6 +4,7 @@ import binascii from typing import Literal, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import ( AllWordsSelectionStrategy, WordSelectionStrategy, @@ -57,6 +58,16 @@ def __init__( self._encoding_func = encoding_func + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with BinAscii converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["encoding_func"] = self._encoding_func + return self._set_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a word using the specified encoding function. diff --git a/pyrit/prompt_converter/binary_converter.py b/pyrit/prompt_converter/binary_converter.py index b1e6f5fe7..877033603 100644 --- a/pyrit/prompt_converter/binary_converter.py +++ b/pyrit/prompt_converter/binary_converter.py @@ -6,6 +6,7 @@ from enum import Enum from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -46,6 +47,16 @@ def __init__( raise TypeError("bits_per_char must be an instance of BinaryConverter.BitsPerChar Enum.") self.bits_per_char = bits_per_char + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with binary converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["bits_per_char"] = self.bits_per_char.value + return self._set_identifier(converter_specific_params=base_params) + def validate_input(self, prompt: str) -> None: """ Check if ``bits_per_char`` is sufficient for the characters in the prompt. diff --git a/pyrit/prompt_converter/caesar_converter.py b/pyrit/prompt_converter/caesar_converter.py index b28b0ad3e..80b717051 100644 --- a/pyrit/prompt_converter/caesar_converter.py +++ b/pyrit/prompt_converter/caesar_converter.py @@ -5,6 +5,7 @@ import string from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -45,6 +46,19 @@ def __init__(self, *, caesar_offset: int, append_description: bool = False) -> N "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with Caesar cipher parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "caesar_offset": self.caesar_offset, + "append_description": self.append_description, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the Caesar cipher. diff --git a/pyrit/prompt_converter/charswap_attack_converter.py b/pyrit/prompt_converter/charswap_attack_converter.py index 1202f4493..51ccd087d 100644 --- a/pyrit/prompt_converter/charswap_attack_converter.py +++ b/pyrit/prompt_converter/charswap_attack_converter.py @@ -5,6 +5,7 @@ import string from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import ( WordProportionSelectionStrategy, WordSelectionStrategy, @@ -47,7 +48,19 @@ def __init__( if max_iterations <= 0: raise ValueError("max_iterations must be greater than 0") - self.max_iterations = max_iterations + self._max_iterations = max_iterations + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with charswap parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "max_iterations": self._max_iterations, + }, + ) async def convert_word_async(self, word: str) -> str: """ @@ -73,7 +86,7 @@ def _perturb_word(self, word: str) -> str: """ if word not in string.punctuation and len(word) > 3: idx_elements = list(word) - for _ in range(self.max_iterations): + for _ in range(self._max_iterations): idx1 = random.randint(1, len(word) - 2) # Swap characters idx_elements[idx1], idx_elements[idx1 + 1] = ( diff --git a/pyrit/prompt_converter/codechameleon_converter.py b/pyrit/prompt_converter/codechameleon_converter.py index cb4b8af1d..f04172e53 100644 --- a/pyrit/prompt_converter/codechameleon_converter.py +++ b/pyrit/prompt_converter/codechameleon_converter.py @@ -9,6 +9,7 @@ from typing import Any, Callable, Optional from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -98,6 +99,20 @@ def __init__( '"reverse", "binary_tree", "odd_even" or "length".' ) + self._encrypt_type = encrypt_type + + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with encryption type. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "encrypt_type": self._encrypt_type, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by applying the specified encryption function. diff --git a/pyrit/prompt_converter/colloquial_wordswap_converter.py b/pyrit/prompt_converter/colloquial_wordswap_converter.py index 7fd5c842d..60d71ae9b 100644 --- a/pyrit/prompt_converter/colloquial_wordswap_converter.py +++ b/pyrit/prompt_converter/colloquial_wordswap_converter.py @@ -5,6 +5,7 @@ import re from typing import Dict, List, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -51,6 +52,19 @@ def __init__( self._colloquial_substitutions = custom_substitutions if custom_substitutions else default_substitutions self._deterministic = deterministic + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with colloquial wordswap parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "deterministic": self._deterministic, + "substitution_keys": sorted(self._colloquial_substitutions.keys()), + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by replacing words with colloquial Singaporean terms. diff --git a/pyrit/prompt_converter/diacritic_converter.py b/pyrit/prompt_converter/diacritic_converter.py index 64ebaa753..a550e4ea4 100644 --- a/pyrit/prompt_converter/diacritic_converter.py +++ b/pyrit/prompt_converter/diacritic_converter.py @@ -4,6 +4,7 @@ import logging import unicodedata +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -43,6 +44,19 @@ def __init__(self, target_chars: str = "aeiou", accent: str = "acute"): self._target_chars = set(target_chars) self._accent = accent + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with diacritic parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "target_chars": sorted(self._target_chars), + "accent": self._accent, + } + ) + def _get_accent_mark(self) -> str: """ Retrieve the Unicode character for the specified diacritic accent. diff --git a/pyrit/prompt_converter/first_letter_converter.py b/pyrit/prompt_converter/first_letter_converter.py index b3bc03266..3b26e0450 100644 --- a/pyrit/prompt_converter/first_letter_converter.py +++ b/pyrit/prompt_converter/first_letter_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -30,6 +31,16 @@ def __init__( super().__init__(word_selection_strategy=word_selection_strategy, word_split_separator=None) self.letter_separator = letter_separator + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with first letter converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["letter_separator"] = self.letter_separator + return self._set_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a single word into the target format supported by the converter. diff --git a/pyrit/prompt_converter/human_in_the_loop_converter.py b/pyrit/prompt_converter/human_in_the_loop_converter.py index 72398cb6f..240b1bb55 100644 --- a/pyrit/prompt_converter/human_in_the_loop_converter.py +++ b/pyrit/prompt_converter/human_in_the_loop_converter.py @@ -4,6 +4,7 @@ import logging from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -33,6 +34,14 @@ def __init__( """ self._converters = converters or [] + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with sub-converters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier(sub_converters=self._converters) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by allowing user interaction before sending it to a target. diff --git a/pyrit/prompt_converter/image_compression_converter.py b/pyrit/prompt_converter/image_compression_converter.py index aa4995695..137c17338 100644 --- a/pyrit/prompt_converter/image_compression_converter.py +++ b/pyrit/prompt_converter/image_compression_converter.py @@ -10,6 +10,7 @@ import aiohttp from PIL import Image +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -123,6 +124,24 @@ def __init__( "Using quality > 95 for JPEG may result in larger files. Consider using a lower quality setting." ) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with image compression parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "output_format": self._output_format, + "quality": self._quality, + "optimize": self._optimize, + "progressive": self._progressive, + "compress_level": self._compress_level, + "lossless": self._lossless, + "method": self._method, + } + ) + def _should_compress(self, original_size: int) -> bool: """ Determine if image should be compressed. diff --git a/pyrit/prompt_converter/insert_punctuation_converter.py b/pyrit/prompt_converter/insert_punctuation_converter.py index 885fb64da..f14062ccb 100644 --- a/pyrit/prompt_converter/insert_punctuation_converter.py +++ b/pyrit/prompt_converter/insert_punctuation_converter.py @@ -6,6 +6,7 @@ import string from typing import List, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -44,6 +45,19 @@ def __init__(self, word_swap_ratio: float = 0.2, between_words: bool = True) -> self._word_swap_ratio = word_swap_ratio self._between_words = between_words + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with punctuation insertion parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "word_swap_ratio": self._word_swap_ratio, + "between_words": self._between_words, + } + ) + def _is_valid_punctuation(self, punctuation_list: List[str]) -> bool: """ Check if all items in the list are valid punctuation characters in string.punctuation. diff --git a/pyrit/prompt_converter/leetspeak_converter.py b/pyrit/prompt_converter/leetspeak_converter.py index 913bb2110..1dde81a7d 100644 --- a/pyrit/prompt_converter/leetspeak_converter.py +++ b/pyrit/prompt_converter/leetspeak_converter.py @@ -4,6 +4,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -49,6 +50,29 @@ def __init__( # Use custom substitutions if provided, otherwise default to the standard ones self._leet_substitutions = custom_substitutions if custom_substitutions else default_substitutions self._deterministic = deterministic + self._has_custom_substitutions = custom_substitutions is not None + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with leetspeak parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + import hashlib + import json + + # Hash custom substitutions if provided + substitutions_hash = None + if self._has_custom_substitutions: + substitutions_str = json.dumps(self._leet_substitutions, sort_keys=True) + substitutions_hash = hashlib.sha256(substitutions_str.encode("utf-8")).hexdigest()[:16] + + return self._set_identifier( + converter_specific_params={ + "deterministic": self._deterministic, + "custom_substitutions_hash": substitutions_hash, + }, + ) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/llm_generic_text_converter.py b/pyrit/prompt_converter/llm_generic_text_converter.py index 8a8a3a361..d34b8dc8b 100644 --- a/pyrit/prompt_converter/llm_generic_text_converter.py +++ b/pyrit/prompt_converter/llm_generic_text_converter.py @@ -1,11 +1,13 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import hashlib import logging import uuid from typing import Any, Optional from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -61,6 +63,33 @@ def __init__( self._user_prompt_template_with_objective = user_prompt_template_with_objective + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with LLM and template parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + # Hash templates if they exist and have a value attribute + system_prompt_hash = None + if self._system_prompt_template and hasattr(self._system_prompt_template, "value"): + system_prompt_hash = hashlib.sha256(str(self._system_prompt_template.value).encode("utf-8")).hexdigest()[ + :16 + ] + + user_prompt_hash = None + if self._user_prompt_template_with_objective and hasattr(self._user_prompt_template_with_objective, "value"): + user_prompt_hash = hashlib.sha256( + str(self._user_prompt_template_with_objective.value).encode("utf-8") + ).hexdigest()[:16] + + return self._set_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "system_prompt_template_hash": system_prompt_hash, + "user_prompt_template_hash": user_prompt_hash, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using an LLM via the specified converter target. diff --git a/pyrit/prompt_converter/math_obfuscation_converter.py b/pyrit/prompt_converter/math_obfuscation_converter.py index cc4224d8c..a594bb97f 100644 --- a/pyrit/prompt_converter/math_obfuscation_converter.py +++ b/pyrit/prompt_converter/math_obfuscation_converter.py @@ -5,6 +5,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -88,6 +89,19 @@ def __init__( self._suffix = suffix if suffix is not None else self.DEFAULT_SUFFIX self._rng = rng or random.Random() + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with math obfuscation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "min_n": self._min_n, + "max_n": self._max_n, + } + ) + async def convert_async( self, *, diff --git a/pyrit/prompt_converter/morse_converter.py b/pyrit/prompt_converter/morse_converter.py index ff8548b9e..cd9a5328f 100644 --- a/pyrit/prompt_converter/morse_converter.py +++ b/pyrit/prompt_converter/morse_converter.py @@ -4,6 +4,7 @@ import pathlib from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -34,6 +35,18 @@ def __init__(self, *, append_description: bool = False) -> None: "then use the chainsaw to cut down the stop sign." ) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with morse converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "append_description": self.append_description, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to morse code. diff --git a/pyrit/prompt_converter/noise_converter.py b/pyrit/prompt_converter/noise_converter.py index a3c2a3b55..0028153d4 100644 --- a/pyrit/prompt_converter/noise_converter.py +++ b/pyrit/prompt_converter/noise_converter.py @@ -8,6 +8,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -60,3 +61,19 @@ def __init__( noise=noise, number_errors=str(number_errors), ) + self._noise = noise + self._number_errors = number_errors + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with noise parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "noise": self._noise, + "number_errors": self._number_errors, + }, + ) diff --git a/pyrit/prompt_converter/pdf_converter.py b/pyrit/prompt_converter/pdf_converter.py index 9ff4fab80..6e26f3980 100644 --- a/pyrit/prompt_converter/pdf_converter.py +++ b/pyrit/prompt_converter/pdf_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import ast +import hashlib from io import BytesIO from pathlib import Path from typing import Any, Dict, List, Optional @@ -12,6 +13,7 @@ from reportlab.pdfgen import canvas from pyrit.common.logger import logger +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt, data_serializer_factory from pyrit.models.data_type_serializer import DataTypeSerializer from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -103,6 +105,31 @@ def __init__( if not all(isinstance(item, dict) for item in self._injection_items): raise ValueError("Each injection item must be a dictionary.") + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with PDF converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + template_hash = None + if self._prompt_template: + template_hash = hashlib.sha256(str(self._prompt_template.value).encode("utf-8")).hexdigest()[:16] + + existing_pdf_path = None + if self._existing_pdf_path: + existing_pdf_path = str(self._existing_pdf_path) + + return self._set_identifier( + converter_specific_params={ + "font_type": self._font_type, + "font_size": self._font_size, + "page_width": self._page_width, + "page_height": self._page_height, + "prompt_template_hash": template_hash, + "existing_pdf_path": existing_pdf_path, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt into a PDF. diff --git a/pyrit/prompt_converter/persuasion_converter.py b/pyrit/prompt_converter/persuasion_converter.py index 4321ff0eb..26ab4c87d 100644 --- a/pyrit/prompt_converter/persuasion_converter.py +++ b/pyrit/prompt_converter/persuasion_converter.py @@ -13,6 +13,7 @@ pyrit_json_retry, remove_markdown_json, ) +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -77,6 +78,20 @@ def __init__( except FileNotFoundError: raise ValueError(f"Persuasion technique '{persuasion_technique}' does not exist or is not supported.") self.system_prompt = str(prompt_template.value) + self._persuasion_technique = persuasion_technique + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with persuasion parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self.converter_target, + converter_specific_params={ + "persuasion_technique": self._persuasion_technique, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ diff --git a/pyrit/prompt_converter/prompt_converter.py b/pyrit/prompt_converter/prompt_converter.py index cfa2d34b1..240ae3e9b 100644 --- a/pyrit/prompt_converter/prompt_converter.py +++ b/pyrit/prompt_converter/prompt_converter.py @@ -165,63 +165,57 @@ async def _replace_text_match(self, match: str) -> ConverterResult: result = await self.convert_async(prompt=match, input_type="text") return result - def get_identifier(self) -> ConverterIdentifier: + def _build_identifier(self) -> ConverterIdentifier: """ - Get the converter identifier. Built lazily on first access. - - Returns: - ConverterIdentifier: The identifier containing all configuration parameters. - """ - if self._identifier is None: - self._build_identifier() - assert self._identifier is not None, "_build_identifier must set _identifier" - return self._identifier - - def _build_identifier(self) -> None: - """ - Build the identifier for this converter. + Build and return the identifier for this converter. Subclasses can override this method to add converter-specific parameters by calling _set_identifier with additional arguments. The default implementation calls _set_identifier with no extra parameters. + + Returns: + ConverterIdentifier: The constructed identifier. """ - self._set_identifier() + return self._set_identifier() def _set_identifier( self, *, sub_converters: Optional[Sequence["PromptConverter"]] = None, - prompt_target: Optional[Any] = None, + converter_target: Optional[Any] = None, converter_specific_params: Optional[Dict[str, Any]] = None, - ) -> None: + ) -> ConverterIdentifier: """ - Construct the converter identifier. + Construct and return the converter identifier. Args: sub_converters: List of sub-converters for composite converters (e.g., ConverterPipeline). Defaults to None. - prompt_target: The prompt target used by this converter (for LLM-based converters). + converter_target: The prompt target used by this converter (for LLM-based converters). Defaults to None. converter_specific_params: Additional converter-specific parameters. Defaults to None. + + Returns: + ConverterIdentifier: The constructed identifier. """ # Build sub_identifier from sub_converters sub_identifier: Optional[List[ConverterIdentifier]] = None if sub_converters: sub_identifier = [converter.get_identifier() for converter in sub_converters] - # Extract target_info from prompt_target + # Extract target_info from converter_target target_info: Optional[Dict[str, Any]] = None - if prompt_target: - target_id = prompt_target.get_identifier() + if converter_target: + target_id = converter_target.get_identifier() # Extract standard fields for converter identification target_info = {} for key in ["__type__", "model_name", "temperature", "top_p"]: if key in target_id: target_info[key] = target_id[key] - self._identifier = ConverterIdentifier( + return ConverterIdentifier( class_name=self.__class__.__name__, class_module=self.__class__.__module__, class_description=self.__class__.__doc__ or "", diff --git a/pyrit/prompt_converter/qr_code_converter.py b/pyrit/prompt_converter/qr_code_converter.py index 10e20f771..fe72a5e8a 100644 --- a/pyrit/prompt_converter/qr_code_converter.py +++ b/pyrit/prompt_converter/qr_code_converter.py @@ -5,6 +5,7 @@ import segno +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -60,6 +61,21 @@ def __init__( self._border_color = border_color or light_color self._img_serializer = data_serializer_factory(category="prompt-memory-entries", data_type="image_path") + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with QR code parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "scale": self._scale, + "border": self._border, + "dark_color": self._dark_color, + "light_color": self._light_color, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt to a QR code image. diff --git a/pyrit/prompt_converter/random_capital_letters_converter.py b/pyrit/prompt_converter/random_capital_letters_converter.py index a86f4b8a9..1c12eb81a 100644 --- a/pyrit/prompt_converter/random_capital_letters_converter.py +++ b/pyrit/prompt_converter/random_capital_letters_converter.py @@ -4,6 +4,7 @@ import logging import random +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -26,6 +27,18 @@ def __init__(self, percentage: float = 100.0) -> None: """ self.percentage = percentage + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with random capital letters parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "percentage": self.percentage, + } + ) + def is_percentage(self, input_string: float) -> bool: """ Check if the input string is a valid percentage between 1 and 100. diff --git a/pyrit/prompt_converter/repeat_token_converter.py b/pyrit/prompt_converter/repeat_token_converter.py index 43d2548dc..c9ac1121b 100644 --- a/pyrit/prompt_converter/repeat_token_converter.py +++ b/pyrit/prompt_converter/repeat_token_converter.py @@ -4,6 +4,7 @@ import re from typing import Literal, Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -46,8 +47,9 @@ def __init__( token_insert_mode (str, optional): The mode of insertion for the repeated token. Can be "split", "prepend", "append", or "repeat". """ - self.token_to_repeat = " " + token_to_repeat.strip() - self.times_to_repeat = times_to_repeat + self._token_to_repeat = " " + token_to_repeat.strip() + self._times_to_repeat = times_to_repeat + self._token_insert_mode = token_insert_mode if token_insert_mode else "split" if not token_insert_mode: token_insert_mode = "split" @@ -80,6 +82,20 @@ def insert(text: str) -> list[str]: self.insert = insert + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with repeat token parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "token_to_repeat": self._token_to_repeat.strip(), + "times_to_repeat": self._times_to_repeat, + "token_insert_mode": self._token_insert_mode, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by repeating the specified token a specified number of times. @@ -99,6 +115,6 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text prompt_parts = self.insert(prompt) return ConverterResult( - output_text=f"{prompt_parts[0]}{self.token_to_repeat * self.times_to_repeat}{prompt_parts[1]}", + output_text=f"{prompt_parts[0]}{self._token_to_repeat * self._times_to_repeat}{prompt_parts[1]}", output_type="text", ) diff --git a/pyrit/prompt_converter/search_replace_converter.py b/pyrit/prompt_converter/search_replace_converter.py index 5f1ec8feb..957660893 100644 --- a/pyrit/prompt_converter/search_replace_converter.py +++ b/pyrit/prompt_converter/search_replace_converter.py @@ -4,6 +4,7 @@ import random import re +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -26,10 +27,22 @@ def __init__(self, pattern: str, replace: str | list[str], regex_flags: int = 0) If a list is provided, a random element will be chosen for replacement. regex_flags (int): Regex flags to use for the replacement. Defaults to 0 (no flags). """ - self.pattern = pattern - self.replace_list = [replace] if isinstance(replace, str) else replace + self._pattern = pattern + self._replace_list = [replace] if isinstance(replace, str) else replace + self._regex_flags = regex_flags - self.regex_flags = regex_flags + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with search/replace parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "pattern": self._pattern, + "replace_list": self._replace_list, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -48,8 +61,8 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - replace = random.choice(self.replace_list) + replace = random.choice(self._replace_list) return ConverterResult( - output_text=re.sub(self.pattern, replace, prompt, flags=self.regex_flags), output_type="text" + output_text=re.sub(self._pattern, replace, prompt, flags=self._regex_flags), output_type="text" ) diff --git a/pyrit/prompt_converter/selective_text_converter.py b/pyrit/prompt_converter/selective_text_converter.py index 093348ddd..d233fdf12 100644 --- a/pyrit/prompt_converter/selective_text_converter.py +++ b/pyrit/prompt_converter/selective_text_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter from pyrit.prompt_converter.text_selection_strategy import ( @@ -89,6 +90,22 @@ def __init__( self._is_word_level = isinstance(selection_strategy, WordSelectionStrategy) self._is_token_based = isinstance(selection_strategy, TokenSelectionStrategy) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with selective text converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + sub_converters=[self._converter], + converter_specific_params={ + "selection_strategy": self._selection_strategy.__class__.__name__, + "preserve_tokens": self._preserve_tokens, + "start_token": self._start_token, + "end_token": self._end_token, + }, + ) + def _validate_converter( self, *, diff --git a/pyrit/prompt_converter/string_join_converter.py b/pyrit/prompt_converter/string_join_converter.py index 3122bf02b..2641cbf39 100644 --- a/pyrit/prompt_converter/string_join_converter.py +++ b/pyrit/prompt_converter/string_join_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -27,7 +28,19 @@ def __init__( If None, all words will be converted. """ super().__init__(word_selection_strategy=word_selection_strategy) - self.join_value = join_value + self._join_value = join_value + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with join parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "join_value": self._join_value, + }, + ) async def convert_word_async(self, word: str) -> str: """ @@ -39,4 +52,4 @@ async def convert_word_async(self, word: str) -> str: Returns: str: The converted word. """ - return self.join_value.join(word) + return self._join_value.join(word) diff --git a/pyrit/prompt_converter/suffix_append_converter.py b/pyrit/prompt_converter/suffix_append_converter.py index 86a8c4323..b1bc36632 100644 --- a/pyrit/prompt_converter/suffix_append_converter.py +++ b/pyrit/prompt_converter/suffix_append_converter.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -29,7 +30,19 @@ def __init__(self, *, suffix: str): if not suffix: raise ValueError("Please specify a suffix (str) to be appended to the prompt.") - self.suffix = suffix + self._suffix = suffix + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with suffix parameter. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "suffix": self._suffix, + }, + ) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ @@ -48,4 +61,4 @@ async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text if not self.input_supported(input_type): raise ValueError("Input type not supported") - return ConverterResult(output_text=prompt + " " + self.suffix, output_type="text") + return ConverterResult(output_text=prompt + " " + self._suffix, output_type="text") diff --git a/pyrit/prompt_converter/template_segment_converter.py b/pyrit/prompt_converter/template_segment_converter.py index 737275108..c725a5789 100644 --- a/pyrit/prompt_converter/template_segment_converter.py +++ b/pyrit/prompt_converter/template_segment_converter.py @@ -1,12 +1,14 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import hashlib import logging import pathlib import random from typing import Optional from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, SeedPrompt from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -69,6 +71,20 @@ def __init__( f"Template parameters: {self.prompt_template.parameters}" ) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with template parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + template_hash = hashlib.sha256(str(self.prompt_template.value).encode("utf-8")).hexdigest()[:16] + return self._set_identifier( + converter_specific_params={ + "template_hash": template_hash, + "number_parameters": self._number_parameters, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by splitting it into random segments and using them to fill the template parameters. diff --git a/pyrit/prompt_converter/tense_converter.py b/pyrit/prompt_converter/tense_converter.py index 32956a865..380534bc1 100644 --- a/pyrit/prompt_converter/tense_converter.py +++ b/pyrit/prompt_converter/tense_converter.py @@ -7,6 +7,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -50,3 +51,17 @@ def __init__( system_prompt_template=prompt_template, tense=tense, ) + self._tense = tense + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with tense parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "tense": self._tense, + }, + ) diff --git a/pyrit/prompt_converter/text_jailbreak_converter.py b/pyrit/prompt_converter/text_jailbreak_converter.py index 3e8f5c969..5140e5eee 100644 --- a/pyrit/prompt_converter/text_jailbreak_converter.py +++ b/pyrit/prompt_converter/text_jailbreak_converter.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. from pyrit.datasets import TextJailBreak +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -23,6 +24,18 @@ def __init__(self, *, jailbreak_template: TextJailBreak): """ self.jail_break_template = jailbreak_template + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with jailbreak template path. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "jailbreak_template_path": self.jail_break_template.template_source, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt using the jailbreak template. diff --git a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py index 3169eeec1..4092d0498 100644 --- a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -32,6 +33,16 @@ def __init__(self, action: Literal["encode", "decode"] = "encode", unicode_tags: self.unicode_tags = unicode_tags super().__init__(action=action) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with ASCII smuggler parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["unicode_tags"] = self.unicode_tags + return self._set_identifier(converter_specific_params=base_params) + def encode_message(self, *, message: str) -> tuple[str, str]: """ Encode the message using Unicode Tags. diff --git a/pyrit/prompt_converter/token_smuggling/base.py b/pyrit/prompt_converter/token_smuggling/base.py index 1c4d00798..0c4fcc287 100644 --- a/pyrit/prompt_converter/token_smuggling/base.py +++ b/pyrit/prompt_converter/token_smuggling/base.py @@ -5,6 +5,7 @@ import logging from typing import Literal, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -36,6 +37,18 @@ def __init__(self, action: Literal["encode", "decode"] = "encode") -> None: raise ValueError("Action must be either 'encode' or 'decode'") self.action = action + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with smuggler action. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "action": self.action, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by either encoding or decoding it based on the specified action. diff --git a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py index 4e9ea1947..e2eb40ac8 100644 --- a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal, Optional, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -42,6 +43,17 @@ def __init__( self.zero_char = zero_char if zero_char is not None else "\u2062" # Invisible Times self.one_char = one_char if one_char is not None else "\u2064" # Invisible Plus + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with sneaky bits parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["zero_char_codepoint"] = hex(ord(self.zero_char)) + base_params["one_char_codepoint"] = hex(ord(self.one_char)) + return self._set_identifier(converter_specific_params=base_params) + def encode_message(self, message: str) -> Tuple[str, str]: """ Encode the message using Sneaky Bits mode. diff --git a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py index ed6ad68ac..175c21557 100644 --- a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py @@ -4,6 +4,7 @@ import logging from typing import Literal, Optional, Tuple +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.token_smuggling.base import SmugglerConverter logger = logging.getLogger(__name__) @@ -51,6 +52,17 @@ def __init__( self.utf8_base_char = base_char_utf8 if base_char_utf8 is not None else "😊" self.embed_in_base = embed_in_base + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with variation selector parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["base_char"] = self.utf8_base_char + base_params["embed_in_base"] = self.embed_in_base + return self._set_identifier(converter_specific_params=base_params) + def encode_message(self, message: str) -> Tuple[str, str]: """ Encode the message using Unicode variation selectors. diff --git a/pyrit/prompt_converter/tone_converter.py b/pyrit/prompt_converter/tone_converter.py index 9b2039846..a561e50a9 100644 --- a/pyrit/prompt_converter/tone_converter.py +++ b/pyrit/prompt_converter/tone_converter.py @@ -7,6 +7,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import SeedPrompt from pyrit.prompt_converter.llm_generic_text_converter import LLMGenericTextConverter from pyrit.prompt_target import PromptChatTarget @@ -53,3 +54,17 @@ def __init__( system_prompt_template=prompt_template, tone=tone, ) + self._tone = tone + + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with tone parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self._converter_target, + converter_specific_params={ + "tone": self._tone, + }, + ) diff --git a/pyrit/prompt_converter/translation_converter.py b/pyrit/prompt_converter/translation_converter.py index 4c44d80cd..e0190e4ac 100644 --- a/pyrit/prompt_converter/translation_converter.py +++ b/pyrit/prompt_converter/translation_converter.py @@ -16,6 +16,7 @@ from pyrit.common.apply_defaults import REQUIRED_VALUE, apply_defaults from pyrit.common.path import CONVERTER_SEED_PROMPT_PATH +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -81,6 +82,19 @@ def __init__( self.system_prompt = prompt_template.render_template_value(languages=language) + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with translation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self.converter_target, + converter_specific_params={ + "language": self.language, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by translating it using the converter target. diff --git a/pyrit/prompt_converter/transparency_attack_converter.py b/pyrit/prompt_converter/transparency_attack_converter.py index e39b17801..7d274504e 100644 --- a/pyrit/prompt_converter/transparency_attack_converter.py +++ b/pyrit/prompt_converter/transparency_attack_converter.py @@ -10,6 +10,7 @@ import numpy from PIL import Image +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType, data_serializer_factory from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -184,6 +185,21 @@ def __init__( self._cached_benign_image = self._load_and_preprocess_image(str(benign_image_path)) + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with transparency attack parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "benign_image_path": str(self.benign_image_path), + "size": self.size, + "steps": self.steps, + "learning_rate": self.learning_rate, + } + ) + def _load_and_preprocess_image(self, path: str) -> numpy.ndarray: # type: ignore[type-arg, unused-ignore] """ Load image, convert to grayscale, resize, and normalize for optimization. diff --git a/pyrit/prompt_converter/unicode_confusable_converter.py b/pyrit/prompt_converter/unicode_confusable_converter.py index 7afd99f3c..84f409258 100644 --- a/pyrit/prompt_converter/unicode_confusable_converter.py +++ b/pyrit/prompt_converter/unicode_confusable_converter.py @@ -9,6 +9,7 @@ from confusable_homoglyphs.confusables import is_confusable from confusables import confusable_characters +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -58,6 +59,19 @@ def __init__( self._source_package = source_package self._deterministic = deterministic + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with unicode confusable parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "source_package": self._source_package, + "deterministic": self._deterministic, + }, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by applying confusable substitutions. This leads to a prompt that looks similar, diff --git a/pyrit/prompt_converter/unicode_replacement_converter.py b/pyrit/prompt_converter/unicode_replacement_converter.py index 70a8442b7..525326877 100644 --- a/pyrit/prompt_converter/unicode_replacement_converter.py +++ b/pyrit/prompt_converter/unicode_replacement_converter.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -29,6 +30,16 @@ def __init__( super().__init__(word_selection_strategy=word_selection_strategy) self.encode_spaces = encode_spaces + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with unicode replacement parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + base_params = super()._build_identifier().converter_specific_params or {} + base_params["encode_spaces"] = self.encode_spaces + return self._set_identifier(converter_specific_params=base_params) + async def convert_word_async(self, word: str) -> str: """ Convert a single word into the target format supported by the converter. diff --git a/pyrit/prompt_converter/unicode_sub_converter.py b/pyrit/prompt_converter/unicode_sub_converter.py index 64a5054e3..64318d7af 100644 --- a/pyrit/prompt_converter/unicode_sub_converter.py +++ b/pyrit/prompt_converter/unicode_sub_converter.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter @@ -22,6 +23,18 @@ def __init__(self, *, start_value: int = 0xE0000) -> None: """ self.startValue = start_value + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with unicode substitution parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "start_value": self.startValue, + } + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by encoding it using any unicode starting point. diff --git a/pyrit/prompt_converter/variation_converter.py b/pyrit/prompt_converter/variation_converter.py index 38ee79362..617241943 100644 --- a/pyrit/prompt_converter/variation_converter.py +++ b/pyrit/prompt_converter/variation_converter.py @@ -15,6 +15,7 @@ pyrit_json_retry, remove_markdown_json, ) +from pyrit.identifiers import ConverterIdentifier from pyrit.models import ( Message, MessagePiece, @@ -67,6 +68,16 @@ def __init__( self.system_prompt = str(prompt_template.render_template_value(number_iterations=str(self.number_variations))) + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with variation parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_target=self.converter_target, + ) + async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ Convert the given prompt by generating variations of it using the converter target. diff --git a/pyrit/prompt_converter/word_level_converter.py b/pyrit/prompt_converter/word_level_converter.py index 2241ca005..def79aeed 100644 --- a/pyrit/prompt_converter/word_level_converter.py +++ b/pyrit/prompt_converter/word_level_converter.py @@ -4,6 +4,7 @@ import abc from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.models import PromptDataType from pyrit.prompt_converter.prompt_converter import ConverterResult, PromptConverter from pyrit.prompt_converter.text_selection_strategy import ( @@ -46,6 +47,19 @@ def __init__( self._word_selection_strategy = word_selection_strategy or AllWordsSelectionStrategy() self._word_split_separator = word_split_separator + def _build_identifier(self) -> ConverterIdentifier: + """Build identifier with word-level converter parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "word_selection_strategy": self._word_selection_strategy.__class__.__name__, + "word_split_separator": self._word_split_separator, + } + ) + @abc.abstractmethod async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/zalgo_converter.py b/pyrit/prompt_converter/zalgo_converter.py index 06a7abbf3..246db9dc1 100644 --- a/pyrit/prompt_converter/zalgo_converter.py +++ b/pyrit/prompt_converter/zalgo_converter.py @@ -5,6 +5,7 @@ import random from typing import Optional +from pyrit.identifiers import ConverterIdentifier from pyrit.prompt_converter.text_selection_strategy import WordSelectionStrategy from pyrit.prompt_converter.word_level_converter import WordLevelConverter @@ -40,6 +41,18 @@ def __init__( self._intensity = self._normalize_intensity(intensity) self._seed = seed + def _build_identifier(self) -> ConverterIdentifier: + """Build the converter identifier with zalgo parameters. + + Returns: + ConverterIdentifier: The identifier for this converter. + """ + return self._set_identifier( + converter_specific_params={ + "intensity": self._intensity, + }, + ) + def _normalize_intensity(self, intensity: int) -> int: try: intensity = int(intensity) diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index a4694478e..81301c7fe 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -6,6 +6,7 @@ from typing import Optional, Type, cast from uuid import UUID +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer import Scorer @@ -195,9 +196,13 @@ def _get_wrapped_scorer(self) -> Scorer: """Return the wrapped scorer.""" return self._wrapped_scorer - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this conversation scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this conversation scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=[self._wrapped_scorer], ) diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 8aab58b92..8f6c66c5c 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -17,6 +17,7 @@ from pyrit.auth import TokenProviderCredential from pyrit.common import default_values +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ( DataTypeSerializer, MessagePiece, @@ -146,9 +147,13 @@ def _category_values(self) -> list[str]: """Get the string values of the configured harm categories for API calls.""" return [category.value for category in self._harm_categories] - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( scorer_specific_params={ "score_categories": self._category_values, } diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index 8d214eea0..607fdaf5a 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -7,6 +7,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SEED_PROMPT_PATH from pyrit.exceptions.exception_classes import InvalidJsonException +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -55,9 +56,13 @@ def __init__( # Render the system prompt with the harm category self._system_prompt = scoring_instructions_template.render_template_value(harm_categories=self._harm_category) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index 3216cfbe5..44aca1b0b 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -7,6 +7,7 @@ import numpy as np +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -55,9 +56,13 @@ def __init__( self.metric = metric self.n = n - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( scorer_specific_params={ "reference_text": self.reference_text, "metric": self.metric.value, diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index b0d53c0bf..11b263b85 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -87,9 +88,13 @@ def __init__( self._metadata_output_key = metadata_output_key self._category_output_key = category_output_key - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index a374d814c..4e7e7f9cf 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -10,6 +10,7 @@ import yaml from pyrit.common.path import HARM_DEFINITION_PATH, SCORER_LIKERT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -189,9 +190,13 @@ def __init__( self._set_likert_scale_system_prompt(likert_scale_path=likert_scale.path) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index f6c5611bf..3ccccfb72 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SCALES_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer @@ -83,9 +84,13 @@ def __init__( self._system_prompt = scoring_instructions_template.render_template_value(**scale_args) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index ae5aca3bc..aa7e3c2e5 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -3,6 +3,7 @@ from typing import List, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_score_aggregator import ( FloatScaleAggregatorFunc, @@ -61,9 +62,13 @@ def __init__( ) self._score_aggregator = score_aggregator - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index 681636606..3a8fab52c 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -4,6 +4,7 @@ import asyncio from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -46,9 +47,13 @@ def __init__( self._rpc_server = AppRPCServer(open_browser=open_browser) self._rpc_server.start() - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index c9fe2a9d5..63027eb27 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -95,18 +95,6 @@ def scorer_type(self) -> ScoreType: else: return "unknown" - def get_identifier(self) -> ScorerIdentifier: - """ - Get the scorer identifier. Built lazily on first access. - - Returns: - ScorerIdentifier: The identifier containing all configuration parameters. - """ - if self._identifier is None: - self._build_identifier() - assert self._identifier is not None, "_build_identifier must set _identifier" - return self._identifier - @property def _memory(self) -> MemoryInterface: return CentralMemory.get_memory_instance() @@ -120,9 +108,9 @@ def _set_identifier( score_aggregator: Optional[str] = None, scorer_specific_params: Optional[Dict[str, Any]] = None, prompt_target: Optional[PromptTarget] = None, - ) -> None: + ) -> ScorerIdentifier: """ - Construct the scorer evaluation identifier. + Construct and return the scorer identifier. Args: system_prompt_template (Optional[str]): The system prompt template used by this scorer. Defaults to None. @@ -132,6 +120,9 @@ def _set_identifier( scorer_specific_params (Optional[Dict[str, Any]]): Additional scorer-specific parameters. Defaults to None. prompt_target (Optional[PromptTarget]): The prompt target used by this scorer. Defaults to None. + + Returns: + ScorerIdentifier: The constructed identifier. """ # Build sub_identifier from sub_scorers (store as dicts for storage) sub_identifier: Optional[List[ScorerIdentifier]] = None @@ -147,7 +138,7 @@ def _set_identifier( if key in target_id: target_info[key] = target_id[key] - self._identifier = ScorerIdentifier( + return ScorerIdentifier( class_name=self.__class__.__name__, class_module=self.__class__.__module__, class_description=self.__class__.__doc__ or "", diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index ebd0d4af4..ceb35a9f9 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -4,6 +4,7 @@ from typing import Optional from pyrit.analytics.text_matching import ExactTextMatching, TextMatching +from pyrit.identifiers import ScorerIdentifier from pyrit.memory.central_memory import CentralMemory from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -51,9 +52,13 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "text_matcher": self._text_matcher.__class__.__name__, diff --git a/pyrit/score/true_false/float_scale_threshold_scorer.py b/pyrit/score/true_false/float_scale_threshold_scorer.py index 76e0b8bec..fab1a7535 100644 --- a/pyrit/score/true_false/float_scale_threshold_scorer.py +++ b/pyrit/score/true_false/float_scale_threshold_scorer.py @@ -4,6 +4,7 @@ import uuid from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.float_scale.float_scale_score_aggregator import ( FloatScaleAggregatorFunc, @@ -54,9 +55,13 @@ def threshold(self) -> float: """Get the threshold value used for score comparison.""" return self._threshold - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 6c6bcdf38..3ee33f6f8 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -9,6 +9,7 @@ from openai import BadRequestError from pyrit.exceptions import PyritException, pyrit_target_retry +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score from pyrit.prompt_target import GandalfLevel, PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -54,9 +55,13 @@ def __init__( self._defender = level.value self._endpoint = "https://gandalf-api.lakera.ai/api/guess-password" - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index a4bbd8e46..52742209b 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -4,6 +4,7 @@ import re from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -42,9 +43,13 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index 6d3c4946d..b6eeb58f0 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -6,6 +6,7 @@ import uuid from typing import Any, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import Message, MessagePiece, Score, ScoreType from pyrit.prompt_target import PromptShieldTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -48,9 +49,13 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index 514b39ad6..fb118cc90 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -50,9 +51,13 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "correct_answer_matching_patterns": self._correct_answer_matching_patterns, diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 3942f5396..4fbc65874 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_CONTENT_CLASSIFIERS_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -77,9 +78,13 @@ def __init__( no_category_found=self._no_category_found_category, ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index 957ef2722..eddc21b53 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -5,6 +5,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -84,9 +85,13 @@ def __init__( self._metadata_output_key = metadata_output_key self._category_output_key = category_output_key - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index 8b3467a1f..a0cdde809 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -5,6 +5,7 @@ from typing import Optional from pyrit.common.path import SCORER_SEED_PROMPT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt, UnvalidatedScore from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -71,9 +72,13 @@ def __init__( self._score_category = ["refusal"] - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt_with_objective, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 1afc533d2..932f53ade 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -9,6 +9,7 @@ from pyrit.common import verify_and_resolve_path from pyrit.common.path import SCORER_SEED_PROMPT_PATH +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score, SeedPrompt from pyrit.prompt_target import PromptChatTarget from pyrit.score.scorer_prompt_validator import ScorerPromptValidator @@ -147,9 +148,13 @@ def __init__( true_description=true_category, false_description=false_category, metadata=metadata ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index 5dc351547..70e5323db 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -4,6 +4,7 @@ from typing import Optional from pyrit.analytics.text_matching import ExactTextMatching, TextMatching +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -50,9 +51,13 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "substring": self._substring, diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 888a36cda..5d7567c19 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -4,6 +4,7 @@ import asyncio from typing import List, Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import TrueFalseAggregatorFunc @@ -52,9 +53,13 @@ def __init__( self._scorers = scorers - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=self._scorers, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/true_false_inverter_scorer.py b/pyrit/score/true_false/true_false_inverter_scorer.py index a8dc3f489..7c4e58e5f 100644 --- a/pyrit/score/true_false/true_false_inverter_scorer.py +++ b/pyrit/score/true_false/true_false_inverter_scorer.py @@ -4,6 +4,7 @@ import uuid from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import ChatMessageRole, Message, MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_scorer import TrueFalseScorer @@ -30,9 +31,13 @@ def __init__(self, *, scorer: TrueFalseScorer, validator: Optional[ScorerPromptV super().__init__(validator=ScorerPromptValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index 6e5fdb483..65668f5d7 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -3,6 +3,7 @@ from typing import Optional +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.scorer_prompt_validator import ScorerPromptValidator from pyrit.score.true_false.true_false_score_aggregator import ( @@ -49,9 +50,13 @@ def __init__( self, validator=validator or self._default_validator, score_aggregator=score_aggregator ) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this scorer.""" - self._set_identifier( + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/tests/unit/registry/test_scorer_registry.py b/tests/unit/registry/test_scorer_registry.py index 5a9d8ce7a..2f29a5886 100644 --- a/tests/unit/registry/test_scorer_registry.py +++ b/tests/unit/registry/test_scorer_registry.py @@ -28,9 +28,13 @@ class MockTrueFalseScorer(TrueFalseScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -48,9 +52,13 @@ class MockFloatScaleScorer(FloatScaleScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -68,9 +76,13 @@ class MockGenericScorer(Scorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index fcd441aa2..e501d3431 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -9,6 +9,7 @@ import pytest from pyrit.exceptions import InvalidJsonException, remove_markdown_json +from pyrit.identifiers import ScorerIdentifier from pyrit.memory import CentralMemory from pyrit.models import Message, MessagePiece, Score from pyrit.prompt_target import PromptChatTarget @@ -60,9 +61,9 @@ class MockScorer(TrueFalseScorer): def __init__(self): super().__init__(validator=DummyValidator()) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + return self._set_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -116,9 +117,9 @@ def __init__(self, *, validator: ScorerPromptValidator): self.scored_piece_ids: list[str] = [] super().__init__(validator=validator) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + return self._set_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: # Track which pieces get scored @@ -1118,9 +1119,9 @@ def __init__(self): self.scored_piece_ids = [] super().__init__(validator=validator) - def _build_identifier(self) -> None: + def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this test scorer.""" - self._set_identifier() + return self._set_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None @@ -1305,8 +1306,8 @@ class TestTrueFalseScorer(TrueFalseScorer): def __init__(self, validator): super().__init__(validator=validator) - def _build_identifier(self) -> None: - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + return self._set_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_true_false_composite_scorer.py b/tests/unit/score/test_true_false_composite_scorer.py index ca0d4472d..ac2a4f72c 100644 --- a/tests/unit/score/test_true_false_composite_scorer.py +++ b/tests/unit/score/test_true_false_composite_scorer.py @@ -41,9 +41,13 @@ def __init__(self, score_value: bool, score_rationale: str, aggregator=None): # Call super().__init__() to properly initialize the scorer including _identifier super().__init__(validator=MagicMock()) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -156,8 +160,8 @@ class InvalidScorer(FloatScaleScorer): def __init__(self): self._validator = MagicMock() - def _build_identifier(self) -> None: - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + return self._set_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_video_scorer.py b/tests/unit/score/test_video_scorer.py index acab4b162..85a68ce1e 100644 --- a/tests/unit/score/test_video_scorer.py +++ b/tests/unit/score/test_video_scorer.py @@ -9,6 +9,7 @@ import numpy as np import pytest +from pyrit.identifiers import ScorerIdentifier from pyrit.models import MessagePiece, Score from pyrit.score.float_scale.float_scale_scorer import FloatScaleScorer from pyrit.score.float_scale.video_float_scale_scorer import VideoFloatScaleScorer @@ -68,9 +69,13 @@ def __init__(self, return_value: bool = True): validator = ScorerPromptValidator(supported_data_types=["image_path"]) super().__init__(validator=validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -96,9 +101,13 @@ def __init__(self, return_value: float = 0.8): validator = ScorerPromptValidator(supported_data_types=["image_path"]) super().__init__(validator=validator) - def _build_identifier(self) -> None: - """Build the scorer evaluation identifier for this mock scorer.""" - self._set_identifier() + def _build_identifier(self) -> ScorerIdentifier: + """Build the scorer evaluation identifier for this mock scorer. + + Returns: + ScorerIdentifier: The identifier for this scorer. + """ + return self._set_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ From 870ccc70c75602e43bd82e37efa481bc784364f2 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 27 Jan 2026 18:40:50 -0800 Subject: [PATCH 4/5] adding tests --- pyrit/memory/memory_models.py | 2 +- .../add_image_text_converter.py | 3 +- .../add_image_to_video_converter.py | 3 +- .../add_text_image_converter.py | 3 +- pyrit/prompt_converter/ascii_art_converter.py | 3 +- pyrit/prompt_converter/atbash_converter.py | 3 +- .../audio_frequency_converter.py | 3 +- .../azure_speech_audio_to_text_converter.py | 3 +- .../azure_speech_text_to_audio_converter.py | 3 +- pyrit/prompt_converter/base64_converter.py | 3 +- pyrit/prompt_converter/bin_ascii_converter.py | 3 +- pyrit/prompt_converter/binary_converter.py | 3 +- pyrit/prompt_converter/caesar_converter.py | 3 +- .../charswap_attack_converter.py | 3 +- .../codechameleon_converter.py | 3 +- .../colloquial_wordswap_converter.py | 3 +- pyrit/prompt_converter/diacritic_converter.py | 3 +- .../first_letter_converter.py | 3 +- .../human_in_the_loop_converter.py | 3 +- .../image_compression_converter.py | 3 +- .../insert_punctuation_converter.py | 3 +- pyrit/prompt_converter/leetspeak_converter.py | 3 +- .../llm_generic_text_converter.py | 3 +- .../math_obfuscation_converter.py | 3 +- pyrit/prompt_converter/morse_converter.py | 3 +- pyrit/prompt_converter/noise_converter.py | 3 +- pyrit/prompt_converter/pdf_converter.py | 3 +- .../prompt_converter/persuasion_converter.py | 3 +- pyrit/prompt_converter/qr_code_converter.py | 3 +- .../random_capital_letters_converter.py | 3 +- .../repeat_token_converter.py | 3 +- .../search_replace_converter.py | 3 +- .../selective_text_converter.py | 3 +- .../prompt_converter/string_join_converter.py | 3 +- .../suffix_append_converter.py | 3 +- .../template_segment_converter.py | 3 +- pyrit/prompt_converter/tense_converter.py | 3 +- .../text_jailbreak_converter.py | 3 +- .../ascii_smuggler_converter.py | 3 +- .../prompt_converter/token_smuggling/base.py | 3 +- .../sneaky_bits_smuggler_converter.py | 3 +- .../variation_selector_smuggler_converter.py | 3 +- pyrit/prompt_converter/tone_converter.py | 3 +- .../prompt_converter/translation_converter.py | 3 +- .../transparency_attack_converter.py | 3 +- .../unicode_confusable_converter.py | 3 +- .../unicode_replacement_converter.py | 3 +- .../prompt_converter/unicode_sub_converter.py | 3 +- pyrit/prompt_converter/variation_converter.py | 3 +- .../prompt_converter/word_level_converter.py | 3 +- pyrit/prompt_converter/zalgo_converter.py | 3 +- pyrit/score/conversation_scorer.py | 9 +- .../azure_content_filter_scorer.py | 3 +- .../score/float_scale/insecure_code_scorer.py | 3 +- pyrit/score/float_scale/plagiarism_scorer.py | 3 +- .../self_ask_general_float_scale_scorer.py | 3 +- .../float_scale/self_ask_likert_scorer.py | 3 +- .../float_scale/self_ask_scale_scorer.py | 3 +- .../float_scale/video_float_scale_scorer.py | 3 +- pyrit/score/human/human_in_the_loop_gradio.py | 3 +- pyrit/score/true_false/decoding_scorer.py | 3 +- .../float_scale_threshold_scorer.py | 3 +- pyrit/score/true_false/gandalf_scorer.py | 3 +- pyrit/score/true_false/markdown_injection.py | 3 +- .../score/true_false/prompt_shield_scorer.py | 3 +- .../true_false/question_answer_scorer.py | 3 +- .../true_false/self_ask_category_scorer.py | 3 +- .../self_ask_general_true_false_scorer.py | 3 +- .../true_false/self_ask_refusal_scorer.py | 3 +- .../true_false/self_ask_true_false_scorer.py | 3 +- pyrit/score/true_false/substring_scorer.py | 3 +- .../true_false/true_false_composite_scorer.py | 3 +- .../true_false/true_false_inverter_scorer.py | 3 +- .../true_false/video_true_false_scorer.py | 3 +- .../identifiers/test_converter_identifier.py | 225 ++++++++++++++++++ 75 files changed, 375 insertions(+), 77 deletions(-) create mode 100644 tests/unit/identifiers/test_converter_identifier.py diff --git a/pyrit/memory/memory_models.py b/pyrit/memory/memory_models.py index 2cccbf056..dd787f718 100644 --- a/pyrit/memory/memory_models.py +++ b/pyrit/memory/memory_models.py @@ -230,7 +230,7 @@ def get_message_piece(self) -> MessagePiece: Returns: MessagePiece: The reconstructed message piece with all its data and scores. """ - converter_ids = ( + converter_ids: Optional[List[Union[ConverterIdentifier, Dict[str, str]]]] = ( [ConverterIdentifier.from_dict(c) for c in self.converter_identifiers] if self.converter_identifiers else None diff --git a/pyrit/prompt_converter/add_image_text_converter.py b/pyrit/prompt_converter/add_image_text_converter.py index 63429af2c..831d93854 100644 --- a/pyrit/prompt_converter/add_image_text_converter.py +++ b/pyrit/prompt_converter/add_image_text_converter.py @@ -65,7 +65,8 @@ def __init__( self._y_pos = y_pos def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with image and text parameters. + """ + Build the converter identifier with image and text parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index dc39e5350..2d622a772 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -63,7 +63,8 @@ def __init__( self._video_path = video_path def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with video converter parameters. + """ + Build identifier with video converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/add_text_image_converter.py b/pyrit/prompt_converter/add_text_image_converter.py index 9a62b1aee..af7a01893 100644 --- a/pyrit/prompt_converter/add_text_image_converter.py +++ b/pyrit/prompt_converter/add_text_image_converter.py @@ -66,7 +66,8 @@ def __init__( self._y_pos = y_pos def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with text and image parameters. + """ + Build the converter identifier with text and image parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/ascii_art_converter.py b/pyrit/prompt_converter/ascii_art_converter.py index fc9991880..cd41509fd 100644 --- a/pyrit/prompt_converter/ascii_art_converter.py +++ b/pyrit/prompt_converter/ascii_art_converter.py @@ -26,7 +26,8 @@ def __init__(self, font: str = "rand") -> None: self._font = font def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with font parameter. + """ + Build the converter identifier with font parameter. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/atbash_converter.py b/pyrit/prompt_converter/atbash_converter.py index 4acc94205..70908c34d 100644 --- a/pyrit/prompt_converter/atbash_converter.py +++ b/pyrit/prompt_converter/atbash_converter.py @@ -41,7 +41,8 @@ def __init__(self, *, append_description: bool = False) -> None: ) def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with Atbash cipher parameters. + """ + Build the converter identifier with Atbash cipher parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/audio_frequency_converter.py b/pyrit/prompt_converter/audio_frequency_converter.py index f308d7503..2235f6475 100644 --- a/pyrit/prompt_converter/audio_frequency_converter.py +++ b/pyrit/prompt_converter/audio_frequency_converter.py @@ -44,7 +44,8 @@ def __init__( self._shift_value = shift_value def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with audio frequency parameters. + """ + Build the converter identifier with audio frequency parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py index 97fb929c3..24d14cd03 100644 --- a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py +++ b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py @@ -88,7 +88,8 @@ def __init__( self.done = False def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with speech recognition parameters. + """ + Build identifier with speech recognition parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py index 09defe733..306466db3 100644 --- a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py +++ b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py @@ -95,7 +95,8 @@ def __init__( self._output_format = output_format def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with speech synthesis parameters. + """ + Build identifier with speech synthesis parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/base64_converter.py b/pyrit/prompt_converter/base64_converter.py index f7b25a0b5..76e52bd31 100644 --- a/pyrit/prompt_converter/base64_converter.py +++ b/pyrit/prompt_converter/base64_converter.py @@ -46,7 +46,8 @@ def __init__(self, *, encoding_func: EncodingFunc = "b64encode") -> None: self._encoding_func = encoding_func def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with encoding function. + """ + Build the converter identifier with encoding function. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/bin_ascii_converter.py b/pyrit/prompt_converter/bin_ascii_converter.py index 3e55d4f42..321cc356d 100644 --- a/pyrit/prompt_converter/bin_ascii_converter.py +++ b/pyrit/prompt_converter/bin_ascii_converter.py @@ -59,7 +59,8 @@ def __init__( self._encoding_func = encoding_func def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with BinAscii converter parameters. + """ + Build identifier with BinAscii converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/binary_converter.py b/pyrit/prompt_converter/binary_converter.py index 877033603..0075c3fe7 100644 --- a/pyrit/prompt_converter/binary_converter.py +++ b/pyrit/prompt_converter/binary_converter.py @@ -48,7 +48,8 @@ def __init__( self.bits_per_char = bits_per_char def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with binary converter parameters. + """ + Build identifier with binary converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/caesar_converter.py b/pyrit/prompt_converter/caesar_converter.py index 80b717051..badca2b9e 100644 --- a/pyrit/prompt_converter/caesar_converter.py +++ b/pyrit/prompt_converter/caesar_converter.py @@ -47,7 +47,8 @@ def __init__(self, *, caesar_offset: int, append_description: bool = False) -> N ) def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with Caesar cipher parameters. + """ + Build the converter identifier with Caesar cipher parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/charswap_attack_converter.py b/pyrit/prompt_converter/charswap_attack_converter.py index 51ccd087d..7bca3ffff 100644 --- a/pyrit/prompt_converter/charswap_attack_converter.py +++ b/pyrit/prompt_converter/charswap_attack_converter.py @@ -51,7 +51,8 @@ def __init__( self._max_iterations = max_iterations def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with charswap parameters. + """ + Build the converter identifier with charswap parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/codechameleon_converter.py b/pyrit/prompt_converter/codechameleon_converter.py index f04172e53..adb993b28 100644 --- a/pyrit/prompt_converter/codechameleon_converter.py +++ b/pyrit/prompt_converter/codechameleon_converter.py @@ -102,7 +102,8 @@ def __init__( self._encrypt_type = encrypt_type def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with encryption type. + """ + Build identifier with encryption type. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/colloquial_wordswap_converter.py b/pyrit/prompt_converter/colloquial_wordswap_converter.py index 60d71ae9b..85fa7ed7d 100644 --- a/pyrit/prompt_converter/colloquial_wordswap_converter.py +++ b/pyrit/prompt_converter/colloquial_wordswap_converter.py @@ -53,7 +53,8 @@ def __init__( self._deterministic = deterministic def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with colloquial wordswap parameters. + """ + Build identifier with colloquial wordswap parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/diacritic_converter.py b/pyrit/prompt_converter/diacritic_converter.py index a550e4ea4..c5833fa0b 100644 --- a/pyrit/prompt_converter/diacritic_converter.py +++ b/pyrit/prompt_converter/diacritic_converter.py @@ -45,7 +45,8 @@ def __init__(self, target_chars: str = "aeiou", accent: str = "acute"): self._accent = accent def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with diacritic parameters. + """ + Build identifier with diacritic parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/first_letter_converter.py b/pyrit/prompt_converter/first_letter_converter.py index 3b26e0450..0d1deb6af 100644 --- a/pyrit/prompt_converter/first_letter_converter.py +++ b/pyrit/prompt_converter/first_letter_converter.py @@ -32,7 +32,8 @@ def __init__( self.letter_separator = letter_separator def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with first letter converter parameters. + """ + Build identifier with first letter converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/human_in_the_loop_converter.py b/pyrit/prompt_converter/human_in_the_loop_converter.py index 240b1bb55..96a314869 100644 --- a/pyrit/prompt_converter/human_in_the_loop_converter.py +++ b/pyrit/prompt_converter/human_in_the_loop_converter.py @@ -35,7 +35,8 @@ def __init__( self._converters = converters or [] def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with sub-converters. + """ + Build identifier with sub-converters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/image_compression_converter.py b/pyrit/prompt_converter/image_compression_converter.py index 137c17338..1feadaf51 100644 --- a/pyrit/prompt_converter/image_compression_converter.py +++ b/pyrit/prompt_converter/image_compression_converter.py @@ -125,7 +125,8 @@ def __init__( ) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with image compression parameters. + """ + Build identifier with image compression parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/insert_punctuation_converter.py b/pyrit/prompt_converter/insert_punctuation_converter.py index f14062ccb..2be34ea77 100644 --- a/pyrit/prompt_converter/insert_punctuation_converter.py +++ b/pyrit/prompt_converter/insert_punctuation_converter.py @@ -46,7 +46,8 @@ def __init__(self, word_swap_ratio: float = 0.2, between_words: bool = True) -> self._between_words = between_words def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with punctuation insertion parameters. + """ + Build identifier with punctuation insertion parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/leetspeak_converter.py b/pyrit/prompt_converter/leetspeak_converter.py index 1dde81a7d..2bc9cfa2e 100644 --- a/pyrit/prompt_converter/leetspeak_converter.py +++ b/pyrit/prompt_converter/leetspeak_converter.py @@ -53,7 +53,8 @@ def __init__( self._has_custom_substitutions = custom_substitutions is not None def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with leetspeak parameters. + """ + Build the converter identifier with leetspeak parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/llm_generic_text_converter.py b/pyrit/prompt_converter/llm_generic_text_converter.py index d34b8dc8b..fd5a41029 100644 --- a/pyrit/prompt_converter/llm_generic_text_converter.py +++ b/pyrit/prompt_converter/llm_generic_text_converter.py @@ -64,7 +64,8 @@ def __init__( self._user_prompt_template_with_objective = user_prompt_template_with_objective def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with LLM and template parameters. + """ + Build the converter identifier with LLM and template parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/math_obfuscation_converter.py b/pyrit/prompt_converter/math_obfuscation_converter.py index a594bb97f..52409f0d1 100644 --- a/pyrit/prompt_converter/math_obfuscation_converter.py +++ b/pyrit/prompt_converter/math_obfuscation_converter.py @@ -90,7 +90,8 @@ def __init__( self._rng = rng or random.Random() def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with math obfuscation parameters. + """ + Build identifier with math obfuscation parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/morse_converter.py b/pyrit/prompt_converter/morse_converter.py index cd9a5328f..9608aceb4 100644 --- a/pyrit/prompt_converter/morse_converter.py +++ b/pyrit/prompt_converter/morse_converter.py @@ -36,7 +36,8 @@ def __init__(self, *, append_description: bool = False) -> None: ) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with morse converter parameters. + """ + Build identifier with morse converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/noise_converter.py b/pyrit/prompt_converter/noise_converter.py index 0028153d4..166374691 100644 --- a/pyrit/prompt_converter/noise_converter.py +++ b/pyrit/prompt_converter/noise_converter.py @@ -65,7 +65,8 @@ def __init__( self._number_errors = number_errors def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with noise parameters. + """ + Build the converter identifier with noise parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/pdf_converter.py b/pyrit/prompt_converter/pdf_converter.py index 6e26f3980..8b15bd8df 100644 --- a/pyrit/prompt_converter/pdf_converter.py +++ b/pyrit/prompt_converter/pdf_converter.py @@ -106,7 +106,8 @@ def __init__( raise ValueError("Each injection item must be a dictionary.") def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with PDF converter parameters. + """ + Build identifier with PDF converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/persuasion_converter.py b/pyrit/prompt_converter/persuasion_converter.py index 26ab4c87d..25528eb60 100644 --- a/pyrit/prompt_converter/persuasion_converter.py +++ b/pyrit/prompt_converter/persuasion_converter.py @@ -81,7 +81,8 @@ def __init__( self._persuasion_technique = persuasion_technique def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with persuasion parameters. + """ + Build the converter identifier with persuasion parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/qr_code_converter.py b/pyrit/prompt_converter/qr_code_converter.py index fe72a5e8a..f6d08430a 100644 --- a/pyrit/prompt_converter/qr_code_converter.py +++ b/pyrit/prompt_converter/qr_code_converter.py @@ -62,7 +62,8 @@ def __init__( self._img_serializer = data_serializer_factory(category="prompt-memory-entries", data_type="image_path") def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with QR code parameters. + """ + Build identifier with QR code parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/random_capital_letters_converter.py b/pyrit/prompt_converter/random_capital_letters_converter.py index 1c12eb81a..e6e075e90 100644 --- a/pyrit/prompt_converter/random_capital_letters_converter.py +++ b/pyrit/prompt_converter/random_capital_letters_converter.py @@ -28,7 +28,8 @@ def __init__(self, percentage: float = 100.0) -> None: self.percentage = percentage def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with random capital letters parameters. + """ + Build identifier with random capital letters parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/repeat_token_converter.py b/pyrit/prompt_converter/repeat_token_converter.py index c9ac1121b..60c5fd0f9 100644 --- a/pyrit/prompt_converter/repeat_token_converter.py +++ b/pyrit/prompt_converter/repeat_token_converter.py @@ -83,7 +83,8 @@ def insert(text: str) -> list[str]: self.insert = insert def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with repeat token parameters. + """ + Build the converter identifier with repeat token parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/search_replace_converter.py b/pyrit/prompt_converter/search_replace_converter.py index 957660893..f5fa7266b 100644 --- a/pyrit/prompt_converter/search_replace_converter.py +++ b/pyrit/prompt_converter/search_replace_converter.py @@ -32,7 +32,8 @@ def __init__(self, pattern: str, replace: str | list[str], regex_flags: int = 0) self._regex_flags = regex_flags def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with search/replace parameters. + """ + Build the converter identifier with search/replace parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/selective_text_converter.py b/pyrit/prompt_converter/selective_text_converter.py index d233fdf12..638d56492 100644 --- a/pyrit/prompt_converter/selective_text_converter.py +++ b/pyrit/prompt_converter/selective_text_converter.py @@ -91,7 +91,8 @@ def __init__( self._is_token_based = isinstance(selection_strategy, TokenSelectionStrategy) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with selective text converter parameters. + """ + Build identifier with selective text converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/string_join_converter.py b/pyrit/prompt_converter/string_join_converter.py index 2641cbf39..7704d8769 100644 --- a/pyrit/prompt_converter/string_join_converter.py +++ b/pyrit/prompt_converter/string_join_converter.py @@ -31,7 +31,8 @@ def __init__( self._join_value = join_value def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with join parameters. + """ + Build the converter identifier with join parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/suffix_append_converter.py b/pyrit/prompt_converter/suffix_append_converter.py index b1bc36632..8c5ac5f6f 100644 --- a/pyrit/prompt_converter/suffix_append_converter.py +++ b/pyrit/prompt_converter/suffix_append_converter.py @@ -33,7 +33,8 @@ def __init__(self, *, suffix: str): self._suffix = suffix def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with suffix parameter. + """ + Build the converter identifier with suffix parameter. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/template_segment_converter.py b/pyrit/prompt_converter/template_segment_converter.py index c725a5789..812da1d5d 100644 --- a/pyrit/prompt_converter/template_segment_converter.py +++ b/pyrit/prompt_converter/template_segment_converter.py @@ -72,7 +72,8 @@ def __init__( ) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with template parameters. + """ + Build identifier with template parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/tense_converter.py b/pyrit/prompt_converter/tense_converter.py index 380534bc1..41759baec 100644 --- a/pyrit/prompt_converter/tense_converter.py +++ b/pyrit/prompt_converter/tense_converter.py @@ -54,7 +54,8 @@ def __init__( self._tense = tense def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with tense parameters. + """ + Build the converter identifier with tense parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/text_jailbreak_converter.py b/pyrit/prompt_converter/text_jailbreak_converter.py index 5140e5eee..bc8c4f9c2 100644 --- a/pyrit/prompt_converter/text_jailbreak_converter.py +++ b/pyrit/prompt_converter/text_jailbreak_converter.py @@ -25,7 +25,8 @@ def __init__(self, *, jailbreak_template: TextJailBreak): self.jail_break_template = jailbreak_template def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with jailbreak template path. + """ + Build identifier with jailbreak template path. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py index 4092d0498..79fa96866 100644 --- a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py @@ -34,7 +34,8 @@ def __init__(self, action: Literal["encode", "decode"] = "encode", unicode_tags: super().__init__(action=action) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with ASCII smuggler parameters. + """ + Build identifier with ASCII smuggler parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/token_smuggling/base.py b/pyrit/prompt_converter/token_smuggling/base.py index 0c4fcc287..392562b22 100644 --- a/pyrit/prompt_converter/token_smuggling/base.py +++ b/pyrit/prompt_converter/token_smuggling/base.py @@ -38,7 +38,8 @@ def __init__(self, action: Literal["encode", "decode"] = "encode") -> None: self.action = action def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with smuggler action. + """ + Build identifier with smuggler action. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py index e2eb40ac8..8d68ee248 100644 --- a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py @@ -44,7 +44,8 @@ def __init__( self.one_char = one_char if one_char is not None else "\u2064" # Invisible Plus def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with sneaky bits parameters. + """ + Build identifier with sneaky bits parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py index 175c21557..72a04074f 100644 --- a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py @@ -53,7 +53,8 @@ def __init__( self.embed_in_base = embed_in_base def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with variation selector parameters. + """ + Build identifier with variation selector parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/tone_converter.py b/pyrit/prompt_converter/tone_converter.py index a561e50a9..9fec3b3c1 100644 --- a/pyrit/prompt_converter/tone_converter.py +++ b/pyrit/prompt_converter/tone_converter.py @@ -57,7 +57,8 @@ def __init__( self._tone = tone def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with tone parameters. + """ + Build the converter identifier with tone parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/translation_converter.py b/pyrit/prompt_converter/translation_converter.py index e0190e4ac..870756ffa 100644 --- a/pyrit/prompt_converter/translation_converter.py +++ b/pyrit/prompt_converter/translation_converter.py @@ -83,7 +83,8 @@ def __init__( self.system_prompt = prompt_template.render_template_value(languages=language) def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with translation parameters. + """ + Build the converter identifier with translation parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/transparency_attack_converter.py b/pyrit/prompt_converter/transparency_attack_converter.py index 7d274504e..c76fe1d9d 100644 --- a/pyrit/prompt_converter/transparency_attack_converter.py +++ b/pyrit/prompt_converter/transparency_attack_converter.py @@ -186,7 +186,8 @@ def __init__( self._cached_benign_image = self._load_and_preprocess_image(str(benign_image_path)) def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with transparency attack parameters. + """ + Build identifier with transparency attack parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/unicode_confusable_converter.py b/pyrit/prompt_converter/unicode_confusable_converter.py index 84f409258..08983b672 100644 --- a/pyrit/prompt_converter/unicode_confusable_converter.py +++ b/pyrit/prompt_converter/unicode_confusable_converter.py @@ -60,7 +60,8 @@ def __init__( self._deterministic = deterministic def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with unicode confusable parameters. + """ + Build the converter identifier with unicode confusable parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/unicode_replacement_converter.py b/pyrit/prompt_converter/unicode_replacement_converter.py index 525326877..e3b094e02 100644 --- a/pyrit/prompt_converter/unicode_replacement_converter.py +++ b/pyrit/prompt_converter/unicode_replacement_converter.py @@ -31,7 +31,8 @@ def __init__( self.encode_spaces = encode_spaces def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with unicode replacement parameters. + """ + Build identifier with unicode replacement parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/unicode_sub_converter.py b/pyrit/prompt_converter/unicode_sub_converter.py index 64318d7af..a250eac09 100644 --- a/pyrit/prompt_converter/unicode_sub_converter.py +++ b/pyrit/prompt_converter/unicode_sub_converter.py @@ -24,7 +24,8 @@ def __init__(self, *, start_value: int = 0xE0000) -> None: self.startValue = start_value def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with unicode substitution parameters. + """ + Build identifier with unicode substitution parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/variation_converter.py b/pyrit/prompt_converter/variation_converter.py index 617241943..47b3a6c50 100644 --- a/pyrit/prompt_converter/variation_converter.py +++ b/pyrit/prompt_converter/variation_converter.py @@ -69,7 +69,8 @@ def __init__( self.system_prompt = str(prompt_template.render_template_value(number_iterations=str(self.number_variations))) def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with variation parameters. + """ + Build the converter identifier with variation parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/word_level_converter.py b/pyrit/prompt_converter/word_level_converter.py index def79aeed..1b41d0e24 100644 --- a/pyrit/prompt_converter/word_level_converter.py +++ b/pyrit/prompt_converter/word_level_converter.py @@ -48,7 +48,8 @@ def __init__( self._word_split_separator = word_split_separator def _build_identifier(self) -> ConverterIdentifier: - """Build identifier with word-level converter parameters. + """ + Build identifier with word-level converter parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/prompt_converter/zalgo_converter.py b/pyrit/prompt_converter/zalgo_converter.py index 246db9dc1..f16158dba 100644 --- a/pyrit/prompt_converter/zalgo_converter.py +++ b/pyrit/prompt_converter/zalgo_converter.py @@ -42,7 +42,8 @@ def __init__( self._seed = seed def _build_identifier(self) -> ConverterIdentifier: - """Build the converter identifier with zalgo parameters. + """ + Build the converter identifier with zalgo parameters. Returns: ConverterIdentifier: The identifier for this converter. diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index 81301c7fe..b3d12735f 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -197,11 +197,12 @@ def _get_wrapped_scorer(self) -> Scorer: return self._wrapped_scorer def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this conversation scorer. + """ + Build the scorer evaluation identifier for this conversation scorer. - Returns: - ScorerIdentifier: The identifier for this scorer. - """ + Returns: + ScorerIdentifier: The identifier for this scorer. + """ return self._set_identifier( sub_scorers=[self._wrapped_scorer], ) diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 8f6c66c5c..0982923b9 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -148,7 +148,8 @@ def _category_values(self) -> list[str]: return [category.value for category in self._harm_categories] def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index 607fdaf5a..ea895ad46 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -57,7 +57,8 @@ def __init__( self._system_prompt = scoring_instructions_template.render_template_value(harm_categories=self._harm_category) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index 44aca1b0b..3eac47b95 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -57,7 +57,8 @@ def __init__( self.n = n def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index 11b263b85..69ab22bce 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -89,7 +89,8 @@ def __init__( self._category_output_key = category_output_key def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index 4e7e7f9cf..aa8e1cbc2 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -191,7 +191,8 @@ def __init__( self._set_likert_scale_system_prompt(likert_scale_path=likert_scale.path) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index 3ccccfb72..993a2bad6 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -85,7 +85,8 @@ def __init__( self._system_prompt = scoring_instructions_template.render_template_value(**scale_args) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index aa7e3c2e5..c1a82af1d 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -63,7 +63,8 @@ def __init__( self._score_aggregator = score_aggregator def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index 3a8fab52c..59cf5aaa7 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -48,7 +48,8 @@ def __init__( self._rpc_server.start() def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index ceb35a9f9..5530f8260 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -53,7 +53,8 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/float_scale_threshold_scorer.py b/pyrit/score/true_false/float_scale_threshold_scorer.py index fab1a7535..a61648499 100644 --- a/pyrit/score/true_false/float_scale_threshold_scorer.py +++ b/pyrit/score/true_false/float_scale_threshold_scorer.py @@ -56,7 +56,8 @@ def threshold(self) -> float: return self._threshold def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 3ee33f6f8..69ccfd215 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -56,7 +56,8 @@ def __init__( self._endpoint = "https://gandalf-api.lakera.ai/api/guess-password" def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index 52742209b..8d2a30af2 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -44,7 +44,8 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index b6eeb58f0..16aa39975 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -50,7 +50,8 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index fb118cc90..edf6405d0 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -52,7 +52,8 @@ def __init__( super().__init__(validator=validator or self._default_validator, score_aggregator=score_aggregator) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 4fbc65874..7f73be798 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -79,7 +79,8 @@ def __init__( ) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index eddc21b53..87b70a2bc 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -86,7 +86,8 @@ def __init__( self._category_output_key = category_output_key def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index a0cdde809..a2b10c9bb 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -73,7 +73,8 @@ def __init__( self._score_category = ["refusal"] def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 932f53ade..14a213b5a 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -149,7 +149,8 @@ def __init__( ) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index 70e5323db..68f79f778 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -52,7 +52,8 @@ def __init__( super().__init__(score_aggregator=aggregator, validator=validator or self._default_validator) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 5d7567c19..17b52f314 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -54,7 +54,8 @@ def __init__( self._scorers = scorers def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/true_false_inverter_scorer.py b/pyrit/score/true_false/true_false_inverter_scorer.py index 7c4e58e5f..243eea43b 100644 --- a/pyrit/score/true_false/true_false_inverter_scorer.py +++ b/pyrit/score/true_false/true_false_inverter_scorer.py @@ -32,7 +32,8 @@ def __init__(self, *, scorer: TrueFalseScorer, validator: Optional[ScorerPromptV super().__init__(validator=ScorerPromptValidator()) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index 65668f5d7..07d543a94 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -51,7 +51,8 @@ def __init__( ) def _build_identifier(self) -> ScorerIdentifier: - """Build the scorer evaluation identifier for this scorer. + """ + Build the scorer evaluation identifier for this scorer. Returns: ScorerIdentifier: The identifier for this scorer. diff --git a/tests/unit/identifiers/test_converter_identifier.py b/tests/unit/identifiers/test_converter_identifier.py new file mode 100644 index 000000000..d1cd56bad --- /dev/null +++ b/tests/unit/identifiers/test_converter_identifier.py @@ -0,0 +1,225 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Tests for ConverterIdentifier-specific functionality. + +Note: Base Identifier functionality (hash computation, to_dict/from_dict basics, +frozen/hashable properties) is tested via ScorerIdentifier in test_scorer_identifier.py. +These tests focus on converter-specific fields and behaviors. +""" + +import pytest + +from pyrit.identifiers import ConverterIdentifier + + +class TestConverterIdentifierSpecificFields: + """Test ConverterIdentifier-specific fields: supported_input_types, supported_output_types, converter_specific_params.""" + + def test_supported_input_output_types_stored_as_tuples(self): + """Test that supported_input_types and supported_output_types are stored correctly.""" + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text", "image_path"), + supported_output_types=("text",), + ) + + assert identifier.supported_input_types == ("text", "image_path") + assert identifier.supported_output_types == ("text",) + + def test_converter_specific_params_stored(self): + """Test that converter_specific_params are stored correctly.""" + identifier = ConverterIdentifier( + class_name="CaesarConverter", + class_module="pyrit.prompt_converter.caesar_converter", + class_description="A Caesar cipher converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + converter_specific_params={"shift": 3, "preserve_case": True}, + ) + + assert identifier.converter_specific_params["shift"] == 3 + assert identifier.converter_specific_params["preserve_case"] is True + + def test_sub_identifier_with_nested_converter(self): + """Test that sub_identifier can hold nested ConverterIdentifier.""" + sub_converter = ConverterIdentifier( + class_name="SubConverter", + class_module="pyrit.prompt_converter.sub_converter", + class_description="A sub converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + ) + + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + sub_identifier=[sub_converter], + ) + + assert len(identifier.sub_identifier) == 1 + assert identifier.sub_identifier[0].class_name == "SubConverter" + + +class TestConverterIdentifierHashDifferences: + """Test that converter-specific fields affect hash computation.""" + + def test_hash_different_for_different_input_types(self): + """Test that different input types produce different hashes.""" + base_args = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_output_types": ("text",), + } + + identifier1 = ConverterIdentifier(supported_input_types=("text",), **base_args) + identifier2 = ConverterIdentifier(supported_input_types=("image_path",), **base_args) + + assert identifier1.hash != identifier2.hash + + def test_hash_different_for_different_output_types(self): + """Test that different output types produce different hashes.""" + base_args = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_input_types": ("text",), + } + + identifier1 = ConverterIdentifier(supported_output_types=("text",), **base_args) + identifier2 = ConverterIdentifier(supported_output_types=("image_path",), **base_args) + + assert identifier1.hash != identifier2.hash + + def test_hash_different_for_different_converter_specific_params(self): + """Test that different converter_specific_params produce different hashes.""" + base_args = { + "class_name": "CaesarConverter", + "class_module": "pyrit.prompt_converter.caesar_converter", + "class_description": "A Caesar cipher converter", + "identifier_type": "instance", + "supported_input_types": ("text",), + "supported_output_types": ("text",), + } + + identifier1 = ConverterIdentifier(converter_specific_params={"shift": 3}, **base_args) + identifier2 = ConverterIdentifier(converter_specific_params={"shift": 5}, **base_args) + + assert identifier1.hash != identifier2.hash + + +class TestConverterIdentifierToDict: + """Test to_dict includes converter-specific fields.""" + + def test_to_dict_includes_supported_types(self): + """Test that supported_input_types and supported_output_types are in to_dict.""" + identifier = ConverterIdentifier( + class_name="TestConverter", + class_module="pyrit.prompt_converter.test_converter", + class_description="A test converter", + identifier_type="instance", + supported_input_types=("text", "image_path"), + supported_output_types=("text",), + ) + + result = identifier.to_dict() + + # Tuples remain as tuples in to_dict + assert result["supported_input_types"] == ("text", "image_path") + assert result["supported_output_types"] == ("text",) + + def test_to_dict_includes_converter_specific_params(self): + """Test that converter_specific_params are included in to_dict.""" + identifier = ConverterIdentifier( + class_name="CaesarConverter", + class_module="pyrit.prompt_converter.caesar_converter", + class_description="A Caesar cipher converter", + identifier_type="instance", + supported_input_types=("text",), + supported_output_types=("text",), + converter_specific_params={"shift": 13, "preserve_case": True}, + ) + + result = identifier.to_dict() + + assert result["converter_specific_params"] == {"shift": 13, "preserve_case": True} + + +class TestConverterIdentifierFromDict: + """Test from_dict handles converter-specific fields.""" + + def test_from_dict_converts_lists_to_tuples(self): + """Test that from_dict converts supported_*_types lists to tuples.""" + data = { + "class_name": "TestConverter", + "class_module": "pyrit.prompt_converter.test_converter", + "class_description": "A test converter", + "identifier_type": "instance", + "supported_input_types": ["text", "image_path"], # List from JSON + "supported_output_types": ["text"], # List from JSON + } + + identifier = ConverterIdentifier.from_dict(data) + + # Lists should be converted to tuples + assert identifier.supported_input_types == ("text", "image_path") + assert identifier.supported_output_types == ("text",) + assert isinstance(identifier.supported_input_types, tuple) + assert isinstance(identifier.supported_output_types, tuple) + + def test_from_dict_provides_defaults_for_missing_types(self): + """Test that from_dict provides defaults for missing supported_*_types fields.""" + data = { + "class_name": "LegacyConverter", + "class_module": "pyrit.prompt_converter.legacy", + "class_description": "A legacy converter", + "identifier_type": "instance", + # Missing supported_input_types and supported_output_types + } + + identifier = ConverterIdentifier.from_dict(data) + + # Should provide empty tuples as defaults + assert identifier.supported_input_types == () + assert identifier.supported_output_types == () + + def test_from_dict_with_nested_sub_identifiers(self): + """Test from_dict with nested sub_identifier list creates ConverterIdentifier objects.""" + data = { + "class_name": "PipelineConverter", + "class_module": "pyrit.prompt_converter.pipeline_converter", + "class_description": "A pipeline converter", + "identifier_type": "instance", + "supported_input_types": ["text"], + "supported_output_types": ["text"], + "sub_identifier": [ + { + "class_name": "SubConverter", + "class_module": "pyrit.prompt_converter.sub", + "class_description": "Sub converter", + "identifier_type": "instance", + "supported_input_types": ["text"], + "supported_output_types": ["image_path"], + }, + ], + } + + identifier = ConverterIdentifier.from_dict(data) + + assert len(identifier.sub_identifier) == 1 + assert isinstance(identifier.sub_identifier[0], ConverterIdentifier) + assert identifier.sub_identifier[0].supported_output_types == ("image_path",) + From d62362ce1d66f28e55e80cfe97e2f2b48f216289 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 27 Jan 2026 22:20:08 -0800 Subject: [PATCH 5/5] pr feedback --- pyrit/prompt_converter/add_image_text_converter.py | 2 +- pyrit/prompt_converter/add_image_to_video_converter.py | 2 +- pyrit/prompt_converter/add_text_image_converter.py | 2 +- pyrit/prompt_converter/ascii_art_converter.py | 2 +- pyrit/prompt_converter/atbash_converter.py | 2 +- pyrit/prompt_converter/audio_frequency_converter.py | 2 +- .../azure_speech_audio_to_text_converter.py | 2 +- .../azure_speech_text_to_audio_converter.py | 2 +- pyrit/prompt_converter/base64_converter.py | 2 +- pyrit/prompt_converter/bin_ascii_converter.py | 2 +- pyrit/prompt_converter/binary_converter.py | 2 +- pyrit/prompt_converter/caesar_converter.py | 2 +- pyrit/prompt_converter/charswap_attack_converter.py | 2 +- pyrit/prompt_converter/codechameleon_converter.py | 2 +- pyrit/prompt_converter/colloquial_wordswap_converter.py | 2 +- pyrit/prompt_converter/diacritic_converter.py | 2 +- pyrit/prompt_converter/first_letter_converter.py | 2 +- pyrit/prompt_converter/human_in_the_loop_converter.py | 2 +- pyrit/prompt_converter/image_compression_converter.py | 2 +- pyrit/prompt_converter/insert_punctuation_converter.py | 2 +- pyrit/prompt_converter/leetspeak_converter.py | 2 +- pyrit/prompt_converter/llm_generic_text_converter.py | 2 +- pyrit/prompt_converter/math_obfuscation_converter.py | 2 +- pyrit/prompt_converter/morse_converter.py | 2 +- pyrit/prompt_converter/noise_converter.py | 2 +- pyrit/prompt_converter/pdf_converter.py | 2 +- pyrit/prompt_converter/persuasion_converter.py | 2 +- pyrit/prompt_converter/prompt_converter.py | 8 ++++---- pyrit/prompt_converter/qr_code_converter.py | 2 +- .../prompt_converter/random_capital_letters_converter.py | 2 +- pyrit/prompt_converter/repeat_token_converter.py | 2 +- pyrit/prompt_converter/search_replace_converter.py | 2 +- pyrit/prompt_converter/selective_text_converter.py | 2 +- pyrit/prompt_converter/string_join_converter.py | 2 +- pyrit/prompt_converter/suffix_append_converter.py | 2 +- pyrit/prompt_converter/template_segment_converter.py | 2 +- pyrit/prompt_converter/tense_converter.py | 2 +- pyrit/prompt_converter/text_jailbreak_converter.py | 2 +- .../token_smuggling/ascii_smuggler_converter.py | 2 +- pyrit/prompt_converter/token_smuggling/base.py | 2 +- .../token_smuggling/sneaky_bits_smuggler_converter.py | 2 +- .../variation_selector_smuggler_converter.py | 2 +- pyrit/prompt_converter/tone_converter.py | 2 +- pyrit/prompt_converter/translation_converter.py | 2 +- pyrit/prompt_converter/transparency_attack_converter.py | 2 +- pyrit/prompt_converter/unicode_confusable_converter.py | 2 +- pyrit/prompt_converter/unicode_replacement_converter.py | 2 +- pyrit/prompt_converter/unicode_sub_converter.py | 2 +- pyrit/prompt_converter/variation_converter.py | 2 +- pyrit/prompt_converter/word_level_converter.py | 2 +- pyrit/prompt_converter/zalgo_converter.py | 2 +- pyrit/score/conversation_scorer.py | 2 +- pyrit/score/float_scale/azure_content_filter_scorer.py | 2 +- pyrit/score/float_scale/insecure_code_scorer.py | 2 +- pyrit/score/float_scale/plagiarism_scorer.py | 2 +- .../float_scale/self_ask_general_float_scale_scorer.py | 2 +- pyrit/score/float_scale/self_ask_likert_scorer.py | 2 +- pyrit/score/float_scale/self_ask_scale_scorer.py | 2 +- pyrit/score/float_scale/video_float_scale_scorer.py | 2 +- pyrit/score/human/human_in_the_loop_gradio.py | 2 +- pyrit/score/scorer.py | 2 +- pyrit/score/true_false/decoding_scorer.py | 2 +- pyrit/score/true_false/float_scale_threshold_scorer.py | 2 +- pyrit/score/true_false/gandalf_scorer.py | 2 +- pyrit/score/true_false/markdown_injection.py | 2 +- pyrit/score/true_false/prompt_shield_scorer.py | 2 +- pyrit/score/true_false/question_answer_scorer.py | 2 +- pyrit/score/true_false/self_ask_category_scorer.py | 2 +- .../true_false/self_ask_general_true_false_scorer.py | 2 +- pyrit/score/true_false/self_ask_refusal_scorer.py | 2 +- pyrit/score/true_false/self_ask_true_false_scorer.py | 2 +- pyrit/score/true_false/substring_scorer.py | 2 +- pyrit/score/true_false/true_false_composite_scorer.py | 2 +- pyrit/score/true_false/true_false_inverter_scorer.py | 2 +- pyrit/score/true_false/video_true_false_scorer.py | 2 +- tests/unit/identifiers/test_converter_identifier.py | 3 --- tests/unit/registry/test_scorer_registry.py | 6 +++--- tests/unit/score/test_conversation_history_scorer.py | 6 +++--- tests/unit/score/test_scorer.py | 8 ++++---- tests/unit/score/test_true_false_composite_scorer.py | 4 ++-- tests/unit/score/test_video_scorer.py | 4 ++-- 81 files changed, 92 insertions(+), 95 deletions(-) diff --git a/pyrit/prompt_converter/add_image_text_converter.py b/pyrit/prompt_converter/add_image_text_converter.py index 831d93854..20d3ce326 100644 --- a/pyrit/prompt_converter/add_image_text_converter.py +++ b/pyrit/prompt_converter/add_image_text_converter.py @@ -71,7 +71,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "img_to_add_path": str(self._img_to_add), "font_name": self._font_name, diff --git a/pyrit/prompt_converter/add_image_to_video_converter.py b/pyrit/prompt_converter/add_image_to_video_converter.py index 2d622a772..dd71c5104 100644 --- a/pyrit/prompt_converter/add_image_to_video_converter.py +++ b/pyrit/prompt_converter/add_image_to_video_converter.py @@ -69,7 +69,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "video_path": str(self._video_path), "img_position": self._img_position, diff --git a/pyrit/prompt_converter/add_text_image_converter.py b/pyrit/prompt_converter/add_text_image_converter.py index af7a01893..fdf9039cf 100644 --- a/pyrit/prompt_converter/add_text_image_converter.py +++ b/pyrit/prompt_converter/add_text_image_converter.py @@ -73,7 +73,7 @@ def _build_identifier(self) -> ConverterIdentifier: ConverterIdentifier: The identifier for this converter. """ text_hash = hashlib.sha256(self._text_to_add.encode("utf-8")).hexdigest()[:16] - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "text_to_add_hash": text_hash, "font_name": self._font_name, diff --git a/pyrit/prompt_converter/ascii_art_converter.py b/pyrit/prompt_converter/ascii_art_converter.py index cd41509fd..eae28f531 100644 --- a/pyrit/prompt_converter/ascii_art_converter.py +++ b/pyrit/prompt_converter/ascii_art_converter.py @@ -32,7 +32,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "font": self._font, }, diff --git a/pyrit/prompt_converter/atbash_converter.py b/pyrit/prompt_converter/atbash_converter.py index 70908c34d..f133b9233 100644 --- a/pyrit/prompt_converter/atbash_converter.py +++ b/pyrit/prompt_converter/atbash_converter.py @@ -47,7 +47,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "append_description": self.append_description, }, diff --git a/pyrit/prompt_converter/audio_frequency_converter.py b/pyrit/prompt_converter/audio_frequency_converter.py index 2235f6475..ecd74f028 100644 --- a/pyrit/prompt_converter/audio_frequency_converter.py +++ b/pyrit/prompt_converter/audio_frequency_converter.py @@ -50,7 +50,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "output_format": self._output_format, "shift_value": self._shift_value, diff --git a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py index 24d14cd03..1b9521f65 100644 --- a/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py +++ b/pyrit/prompt_converter/azure_speech_audio_to_text_converter.py @@ -94,7 +94,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "recognition_language": self._recognition_language, } diff --git a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py index 306466db3..105177c02 100644 --- a/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py +++ b/pyrit/prompt_converter/azure_speech_text_to_audio_converter.py @@ -101,7 +101,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "synthesis_language": self._synthesis_language, "synthesis_voice_name": self._synthesis_voice_name, diff --git a/pyrit/prompt_converter/base64_converter.py b/pyrit/prompt_converter/base64_converter.py index 76e52bd31..ccbb685fa 100644 --- a/pyrit/prompt_converter/base64_converter.py +++ b/pyrit/prompt_converter/base64_converter.py @@ -52,7 +52,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "encoding_func": self._encoding_func, }, diff --git a/pyrit/prompt_converter/bin_ascii_converter.py b/pyrit/prompt_converter/bin_ascii_converter.py index 321cc356d..a3c3eac20 100644 --- a/pyrit/prompt_converter/bin_ascii_converter.py +++ b/pyrit/prompt_converter/bin_ascii_converter.py @@ -67,7 +67,7 @@ def _build_identifier(self) -> ConverterIdentifier: """ base_params = super()._build_identifier().converter_specific_params or {} base_params["encoding_func"] = self._encoding_func - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/binary_converter.py b/pyrit/prompt_converter/binary_converter.py index 0075c3fe7..245da36f7 100644 --- a/pyrit/prompt_converter/binary_converter.py +++ b/pyrit/prompt_converter/binary_converter.py @@ -56,7 +56,7 @@ def _build_identifier(self) -> ConverterIdentifier: """ base_params = super()._build_identifier().converter_specific_params or {} base_params["bits_per_char"] = self.bits_per_char.value - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) def validate_input(self, prompt: str) -> None: """ diff --git a/pyrit/prompt_converter/caesar_converter.py b/pyrit/prompt_converter/caesar_converter.py index badca2b9e..b4a7e3816 100644 --- a/pyrit/prompt_converter/caesar_converter.py +++ b/pyrit/prompt_converter/caesar_converter.py @@ -53,7 +53,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "caesar_offset": self.caesar_offset, "append_description": self.append_description, diff --git a/pyrit/prompt_converter/charswap_attack_converter.py b/pyrit/prompt_converter/charswap_attack_converter.py index 7bca3ffff..06534f618 100644 --- a/pyrit/prompt_converter/charswap_attack_converter.py +++ b/pyrit/prompt_converter/charswap_attack_converter.py @@ -57,7 +57,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "max_iterations": self._max_iterations, }, diff --git a/pyrit/prompt_converter/codechameleon_converter.py b/pyrit/prompt_converter/codechameleon_converter.py index adb993b28..c51db730d 100644 --- a/pyrit/prompt_converter/codechameleon_converter.py +++ b/pyrit/prompt_converter/codechameleon_converter.py @@ -108,7 +108,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "encrypt_type": self._encrypt_type, } diff --git a/pyrit/prompt_converter/colloquial_wordswap_converter.py b/pyrit/prompt_converter/colloquial_wordswap_converter.py index 85fa7ed7d..96ab2d6c1 100644 --- a/pyrit/prompt_converter/colloquial_wordswap_converter.py +++ b/pyrit/prompt_converter/colloquial_wordswap_converter.py @@ -59,7 +59,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "deterministic": self._deterministic, "substitution_keys": sorted(self._colloquial_substitutions.keys()), diff --git a/pyrit/prompt_converter/diacritic_converter.py b/pyrit/prompt_converter/diacritic_converter.py index c5833fa0b..08e229e16 100644 --- a/pyrit/prompt_converter/diacritic_converter.py +++ b/pyrit/prompt_converter/diacritic_converter.py @@ -51,7 +51,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "target_chars": sorted(self._target_chars), "accent": self._accent, diff --git a/pyrit/prompt_converter/first_letter_converter.py b/pyrit/prompt_converter/first_letter_converter.py index 0d1deb6af..7449fc8f2 100644 --- a/pyrit/prompt_converter/first_letter_converter.py +++ b/pyrit/prompt_converter/first_letter_converter.py @@ -40,7 +40,7 @@ def _build_identifier(self) -> ConverterIdentifier: """ base_params = super()._build_identifier().converter_specific_params or {} base_params["letter_separator"] = self.letter_separator - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/human_in_the_loop_converter.py b/pyrit/prompt_converter/human_in_the_loop_converter.py index 96a314869..249497242 100644 --- a/pyrit/prompt_converter/human_in_the_loop_converter.py +++ b/pyrit/prompt_converter/human_in_the_loop_converter.py @@ -41,7 +41,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier(sub_converters=self._converters) + return self._create_identifier(sub_converters=self._converters) async def convert_async(self, *, prompt: str, input_type: PromptDataType = "text") -> ConverterResult: """ diff --git a/pyrit/prompt_converter/image_compression_converter.py b/pyrit/prompt_converter/image_compression_converter.py index 1feadaf51..916e8ebee 100644 --- a/pyrit/prompt_converter/image_compression_converter.py +++ b/pyrit/prompt_converter/image_compression_converter.py @@ -131,7 +131,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "output_format": self._output_format, "quality": self._quality, diff --git a/pyrit/prompt_converter/insert_punctuation_converter.py b/pyrit/prompt_converter/insert_punctuation_converter.py index 2be34ea77..91d15378d 100644 --- a/pyrit/prompt_converter/insert_punctuation_converter.py +++ b/pyrit/prompt_converter/insert_punctuation_converter.py @@ -52,7 +52,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "word_swap_ratio": self._word_swap_ratio, "between_words": self._between_words, diff --git a/pyrit/prompt_converter/leetspeak_converter.py b/pyrit/prompt_converter/leetspeak_converter.py index 2bc9cfa2e..5d8cc8c28 100644 --- a/pyrit/prompt_converter/leetspeak_converter.py +++ b/pyrit/prompt_converter/leetspeak_converter.py @@ -68,7 +68,7 @@ def _build_identifier(self) -> ConverterIdentifier: substitutions_str = json.dumps(self._leet_substitutions, sort_keys=True) substitutions_hash = hashlib.sha256(substitutions_str.encode("utf-8")).hexdigest()[:16] - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "deterministic": self._deterministic, "custom_substitutions_hash": substitutions_hash, diff --git a/pyrit/prompt_converter/llm_generic_text_converter.py b/pyrit/prompt_converter/llm_generic_text_converter.py index fd5a41029..b03be4f38 100644 --- a/pyrit/prompt_converter/llm_generic_text_converter.py +++ b/pyrit/prompt_converter/llm_generic_text_converter.py @@ -83,7 +83,7 @@ def _build_identifier(self) -> ConverterIdentifier: str(self._user_prompt_template_with_objective.value).encode("utf-8") ).hexdigest()[:16] - return self._set_identifier( + return self._create_identifier( converter_target=self._converter_target, converter_specific_params={ "system_prompt_template_hash": system_prompt_hash, diff --git a/pyrit/prompt_converter/math_obfuscation_converter.py b/pyrit/prompt_converter/math_obfuscation_converter.py index 52409f0d1..83c559ba9 100644 --- a/pyrit/prompt_converter/math_obfuscation_converter.py +++ b/pyrit/prompt_converter/math_obfuscation_converter.py @@ -96,7 +96,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "min_n": self._min_n, "max_n": self._max_n, diff --git a/pyrit/prompt_converter/morse_converter.py b/pyrit/prompt_converter/morse_converter.py index 9608aceb4..8b837258f 100644 --- a/pyrit/prompt_converter/morse_converter.py +++ b/pyrit/prompt_converter/morse_converter.py @@ -42,7 +42,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "append_description": self.append_description, } diff --git a/pyrit/prompt_converter/noise_converter.py b/pyrit/prompt_converter/noise_converter.py index 166374691..5496e967f 100644 --- a/pyrit/prompt_converter/noise_converter.py +++ b/pyrit/prompt_converter/noise_converter.py @@ -71,7 +71,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self._converter_target, converter_specific_params={ "noise": self._noise, diff --git a/pyrit/prompt_converter/pdf_converter.py b/pyrit/prompt_converter/pdf_converter.py index 8b15bd8df..b84cb23dd 100644 --- a/pyrit/prompt_converter/pdf_converter.py +++ b/pyrit/prompt_converter/pdf_converter.py @@ -120,7 +120,7 @@ def _build_identifier(self) -> ConverterIdentifier: if self._existing_pdf_path: existing_pdf_path = str(self._existing_pdf_path) - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "font_type": self._font_type, "font_size": self._font_size, diff --git a/pyrit/prompt_converter/persuasion_converter.py b/pyrit/prompt_converter/persuasion_converter.py index 25528eb60..7282b43f1 100644 --- a/pyrit/prompt_converter/persuasion_converter.py +++ b/pyrit/prompt_converter/persuasion_converter.py @@ -87,7 +87,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self.converter_target, converter_specific_params={ "persuasion_technique": self._persuasion_technique, diff --git a/pyrit/prompt_converter/prompt_converter.py b/pyrit/prompt_converter/prompt_converter.py index 240ae3e9b..f7e0a6a2c 100644 --- a/pyrit/prompt_converter/prompt_converter.py +++ b/pyrit/prompt_converter/prompt_converter.py @@ -170,16 +170,16 @@ def _build_identifier(self) -> ConverterIdentifier: Build and return the identifier for this converter. Subclasses can override this method to add converter-specific parameters - by calling _set_identifier with additional arguments. + by calling _create_identifier with additional arguments. - The default implementation calls _set_identifier with no extra parameters. + The default implementation calls _create_identifier with no extra parameters. Returns: ConverterIdentifier: The constructed identifier. """ - return self._set_identifier() + return self._create_identifier() - def _set_identifier( + def _create_identifier( self, *, sub_converters: Optional[Sequence["PromptConverter"]] = None, diff --git a/pyrit/prompt_converter/qr_code_converter.py b/pyrit/prompt_converter/qr_code_converter.py index f6d08430a..d4e9c7ea8 100644 --- a/pyrit/prompt_converter/qr_code_converter.py +++ b/pyrit/prompt_converter/qr_code_converter.py @@ -68,7 +68,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "scale": self._scale, "border": self._border, diff --git a/pyrit/prompt_converter/random_capital_letters_converter.py b/pyrit/prompt_converter/random_capital_letters_converter.py index e6e075e90..9e87a30e7 100644 --- a/pyrit/prompt_converter/random_capital_letters_converter.py +++ b/pyrit/prompt_converter/random_capital_letters_converter.py @@ -34,7 +34,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "percentage": self.percentage, } diff --git a/pyrit/prompt_converter/repeat_token_converter.py b/pyrit/prompt_converter/repeat_token_converter.py index 60c5fd0f9..e568c2514 100644 --- a/pyrit/prompt_converter/repeat_token_converter.py +++ b/pyrit/prompt_converter/repeat_token_converter.py @@ -89,7 +89,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "token_to_repeat": self._token_to_repeat.strip(), "times_to_repeat": self._times_to_repeat, diff --git a/pyrit/prompt_converter/search_replace_converter.py b/pyrit/prompt_converter/search_replace_converter.py index f5fa7266b..4439cc66f 100644 --- a/pyrit/prompt_converter/search_replace_converter.py +++ b/pyrit/prompt_converter/search_replace_converter.py @@ -38,7 +38,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "pattern": self._pattern, "replace_list": self._replace_list, diff --git a/pyrit/prompt_converter/selective_text_converter.py b/pyrit/prompt_converter/selective_text_converter.py index 638d56492..393606d96 100644 --- a/pyrit/prompt_converter/selective_text_converter.py +++ b/pyrit/prompt_converter/selective_text_converter.py @@ -97,7 +97,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( sub_converters=[self._converter], converter_specific_params={ "selection_strategy": self._selection_strategy.__class__.__name__, diff --git a/pyrit/prompt_converter/string_join_converter.py b/pyrit/prompt_converter/string_join_converter.py index 7704d8769..ab94c8517 100644 --- a/pyrit/prompt_converter/string_join_converter.py +++ b/pyrit/prompt_converter/string_join_converter.py @@ -37,7 +37,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "join_value": self._join_value, }, diff --git a/pyrit/prompt_converter/suffix_append_converter.py b/pyrit/prompt_converter/suffix_append_converter.py index 8c5ac5f6f..3aff3fda4 100644 --- a/pyrit/prompt_converter/suffix_append_converter.py +++ b/pyrit/prompt_converter/suffix_append_converter.py @@ -39,7 +39,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "suffix": self._suffix, }, diff --git a/pyrit/prompt_converter/template_segment_converter.py b/pyrit/prompt_converter/template_segment_converter.py index 812da1d5d..61defff6d 100644 --- a/pyrit/prompt_converter/template_segment_converter.py +++ b/pyrit/prompt_converter/template_segment_converter.py @@ -79,7 +79,7 @@ def _build_identifier(self) -> ConverterIdentifier: ConverterIdentifier: The identifier for this converter. """ template_hash = hashlib.sha256(str(self.prompt_template.value).encode("utf-8")).hexdigest()[:16] - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "template_hash": template_hash, "number_parameters": self._number_parameters, diff --git a/pyrit/prompt_converter/tense_converter.py b/pyrit/prompt_converter/tense_converter.py index 41759baec..13e2af822 100644 --- a/pyrit/prompt_converter/tense_converter.py +++ b/pyrit/prompt_converter/tense_converter.py @@ -60,7 +60,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self._converter_target, converter_specific_params={ "tense": self._tense, diff --git a/pyrit/prompt_converter/text_jailbreak_converter.py b/pyrit/prompt_converter/text_jailbreak_converter.py index bc8c4f9c2..84d03a6c6 100644 --- a/pyrit/prompt_converter/text_jailbreak_converter.py +++ b/pyrit/prompt_converter/text_jailbreak_converter.py @@ -31,7 +31,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "jailbreak_template_path": self.jail_break_template.template_source, } diff --git a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py index 79fa96866..b90440953 100644 --- a/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/ascii_smuggler_converter.py @@ -42,7 +42,7 @@ def _build_identifier(self) -> ConverterIdentifier: """ base_params = super()._build_identifier().converter_specific_params or {} base_params["unicode_tags"] = self.unicode_tags - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) def encode_message(self, *, message: str) -> tuple[str, str]: """ diff --git a/pyrit/prompt_converter/token_smuggling/base.py b/pyrit/prompt_converter/token_smuggling/base.py index 392562b22..f41d18473 100644 --- a/pyrit/prompt_converter/token_smuggling/base.py +++ b/pyrit/prompt_converter/token_smuggling/base.py @@ -44,7 +44,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "action": self.action, } diff --git a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py index 8d68ee248..26edf77a7 100644 --- a/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/sneaky_bits_smuggler_converter.py @@ -53,7 +53,7 @@ def _build_identifier(self) -> ConverterIdentifier: base_params = super()._build_identifier().converter_specific_params or {} base_params["zero_char_codepoint"] = hex(ord(self.zero_char)) base_params["one_char_codepoint"] = hex(ord(self.one_char)) - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) def encode_message(self, message: str) -> Tuple[str, str]: """ diff --git a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py index 72a04074f..1d3b5ef26 100644 --- a/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py +++ b/pyrit/prompt_converter/token_smuggling/variation_selector_smuggler_converter.py @@ -62,7 +62,7 @@ def _build_identifier(self) -> ConverterIdentifier: base_params = super()._build_identifier().converter_specific_params or {} base_params["base_char"] = self.utf8_base_char base_params["embed_in_base"] = self.embed_in_base - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) def encode_message(self, message: str) -> Tuple[str, str]: """ diff --git a/pyrit/prompt_converter/tone_converter.py b/pyrit/prompt_converter/tone_converter.py index 9fec3b3c1..2c55f1ded 100644 --- a/pyrit/prompt_converter/tone_converter.py +++ b/pyrit/prompt_converter/tone_converter.py @@ -63,7 +63,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self._converter_target, converter_specific_params={ "tone": self._tone, diff --git a/pyrit/prompt_converter/translation_converter.py b/pyrit/prompt_converter/translation_converter.py index 870756ffa..cf6dba7cb 100644 --- a/pyrit/prompt_converter/translation_converter.py +++ b/pyrit/prompt_converter/translation_converter.py @@ -89,7 +89,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self.converter_target, converter_specific_params={ "language": self.language, diff --git a/pyrit/prompt_converter/transparency_attack_converter.py b/pyrit/prompt_converter/transparency_attack_converter.py index c76fe1d9d..419d58572 100644 --- a/pyrit/prompt_converter/transparency_attack_converter.py +++ b/pyrit/prompt_converter/transparency_attack_converter.py @@ -192,7 +192,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "benign_image_path": str(self.benign_image_path), "size": self.size, diff --git a/pyrit/prompt_converter/unicode_confusable_converter.py b/pyrit/prompt_converter/unicode_confusable_converter.py index 08983b672..b9d9790a7 100644 --- a/pyrit/prompt_converter/unicode_confusable_converter.py +++ b/pyrit/prompt_converter/unicode_confusable_converter.py @@ -66,7 +66,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "source_package": self._source_package, "deterministic": self._deterministic, diff --git a/pyrit/prompt_converter/unicode_replacement_converter.py b/pyrit/prompt_converter/unicode_replacement_converter.py index e3b094e02..e5a3862b0 100644 --- a/pyrit/prompt_converter/unicode_replacement_converter.py +++ b/pyrit/prompt_converter/unicode_replacement_converter.py @@ -39,7 +39,7 @@ def _build_identifier(self) -> ConverterIdentifier: """ base_params = super()._build_identifier().converter_specific_params or {} base_params["encode_spaces"] = self.encode_spaces - return self._set_identifier(converter_specific_params=base_params) + return self._create_identifier(converter_specific_params=base_params) async def convert_word_async(self, word: str) -> str: """ diff --git a/pyrit/prompt_converter/unicode_sub_converter.py b/pyrit/prompt_converter/unicode_sub_converter.py index a250eac09..d85272da8 100644 --- a/pyrit/prompt_converter/unicode_sub_converter.py +++ b/pyrit/prompt_converter/unicode_sub_converter.py @@ -30,7 +30,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "start_value": self.startValue, } diff --git a/pyrit/prompt_converter/variation_converter.py b/pyrit/prompt_converter/variation_converter.py index 47b3a6c50..8b9ca98aa 100644 --- a/pyrit/prompt_converter/variation_converter.py +++ b/pyrit/prompt_converter/variation_converter.py @@ -75,7 +75,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_target=self.converter_target, ) diff --git a/pyrit/prompt_converter/word_level_converter.py b/pyrit/prompt_converter/word_level_converter.py index 1b41d0e24..96ed8a066 100644 --- a/pyrit/prompt_converter/word_level_converter.py +++ b/pyrit/prompt_converter/word_level_converter.py @@ -54,7 +54,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "word_selection_strategy": self._word_selection_strategy.__class__.__name__, "word_split_separator": self._word_split_separator, diff --git a/pyrit/prompt_converter/zalgo_converter.py b/pyrit/prompt_converter/zalgo_converter.py index f16158dba..f8a23106d 100644 --- a/pyrit/prompt_converter/zalgo_converter.py +++ b/pyrit/prompt_converter/zalgo_converter.py @@ -48,7 +48,7 @@ def _build_identifier(self) -> ConverterIdentifier: Returns: ConverterIdentifier: The identifier for this converter. """ - return self._set_identifier( + return self._create_identifier( converter_specific_params={ "intensity": self._intensity, }, diff --git a/pyrit/score/conversation_scorer.py b/pyrit/score/conversation_scorer.py index b3d12735f..1b93088cb 100644 --- a/pyrit/score/conversation_scorer.py +++ b/pyrit/score/conversation_scorer.py @@ -203,7 +203,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=[self._wrapped_scorer], ) diff --git a/pyrit/score/float_scale/azure_content_filter_scorer.py b/pyrit/score/float_scale/azure_content_filter_scorer.py index 0982923b9..9b71f20a3 100644 --- a/pyrit/score/float_scale/azure_content_filter_scorer.py +++ b/pyrit/score/float_scale/azure_content_filter_scorer.py @@ -154,7 +154,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( scorer_specific_params={ "score_categories": self._category_values, } diff --git a/pyrit/score/float_scale/insecure_code_scorer.py b/pyrit/score/float_scale/insecure_code_scorer.py index ea895ad46..6086493d3 100644 --- a/pyrit/score/float_scale/insecure_code_scorer.py +++ b/pyrit/score/float_scale/insecure_code_scorer.py @@ -63,7 +63,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/plagiarism_scorer.py b/pyrit/score/float_scale/plagiarism_scorer.py index 3eac47b95..f1a36cd13 100644 --- a/pyrit/score/float_scale/plagiarism_scorer.py +++ b/pyrit/score/float_scale/plagiarism_scorer.py @@ -63,7 +63,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( scorer_specific_params={ "reference_text": self.reference_text, "metric": self.metric.value, diff --git a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py index 69ab22bce..e1d482928 100644 --- a/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_general_float_scale_scorer.py @@ -95,7 +95,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/self_ask_likert_scorer.py b/pyrit/score/float_scale/self_ask_likert_scorer.py index aa8e1cbc2..c01fe6507 100644 --- a/pyrit/score/float_scale/self_ask_likert_scorer.py +++ b/pyrit/score/float_scale/self_ask_likert_scorer.py @@ -197,7 +197,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, ) diff --git a/pyrit/score/float_scale/self_ask_scale_scorer.py b/pyrit/score/float_scale/self_ask_scale_scorer.py index 993a2bad6..5e502681d 100644 --- a/pyrit/score/float_scale/self_ask_scale_scorer.py +++ b/pyrit/score/float_scale/self_ask_scale_scorer.py @@ -91,7 +91,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/float_scale/video_float_scale_scorer.py b/pyrit/score/float_scale/video_float_scale_scorer.py index c1a82af1d..54c81ec1f 100644 --- a/pyrit/score/float_scale/video_float_scale_scorer.py +++ b/pyrit/score/float_scale/video_float_scale_scorer.py @@ -69,7 +69,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/human/human_in_the_loop_gradio.py b/pyrit/score/human/human_in_the_loop_gradio.py index 59cf5aaa7..f9b3226ba 100644 --- a/pyrit/score/human/human_in_the_loop_gradio.py +++ b/pyrit/score/human/human_in_the_loop_gradio.py @@ -54,7 +54,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/scorer.py b/pyrit/score/scorer.py index 63027eb27..61c1d466e 100644 --- a/pyrit/score/scorer.py +++ b/pyrit/score/scorer.py @@ -99,7 +99,7 @@ def scorer_type(self) -> ScoreType: def _memory(self) -> MemoryInterface: return CentralMemory.get_memory_instance() - def _set_identifier( + def _create_identifier( self, *, system_prompt_template: Optional[str] = None, diff --git a/pyrit/score/true_false/decoding_scorer.py b/pyrit/score/true_false/decoding_scorer.py index 5530f8260..8034f5a00 100644 --- a/pyrit/score/true_false/decoding_scorer.py +++ b/pyrit/score/true_false/decoding_scorer.py @@ -59,7 +59,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "text_matcher": self._text_matcher.__class__.__name__, diff --git a/pyrit/score/true_false/float_scale_threshold_scorer.py b/pyrit/score/true_false/float_scale_threshold_scorer.py index a61648499..5ca8e274c 100644 --- a/pyrit/score/true_false/float_scale_threshold_scorer.py +++ b/pyrit/score/true_false/float_scale_threshold_scorer.py @@ -62,7 +62,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/pyrit/score/true_false/gandalf_scorer.py b/pyrit/score/true_false/gandalf_scorer.py index 69ccfd215..932bf49df 100644 --- a/pyrit/score/true_false/gandalf_scorer.py +++ b/pyrit/score/true_false/gandalf_scorer.py @@ -62,7 +62,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/markdown_injection.py b/pyrit/score/true_false/markdown_injection.py index 8d2a30af2..f45fc90f1 100644 --- a/pyrit/score/true_false/markdown_injection.py +++ b/pyrit/score/true_false/markdown_injection.py @@ -50,7 +50,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/prompt_shield_scorer.py b/pyrit/score/true_false/prompt_shield_scorer.py index 16aa39975..edfc697b1 100644 --- a/pyrit/score/true_false/prompt_shield_scorer.py +++ b/pyrit/score/true_false/prompt_shield_scorer.py @@ -56,7 +56,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/question_answer_scorer.py b/pyrit/score/true_false/question_answer_scorer.py index edf6405d0..63ba8c6c1 100644 --- a/pyrit/score/true_false/question_answer_scorer.py +++ b/pyrit/score/true_false/question_answer_scorer.py @@ -58,7 +58,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "correct_answer_matching_patterns": self._correct_answer_matching_patterns, diff --git a/pyrit/score/true_false/self_ask_category_scorer.py b/pyrit/score/true_false/self_ask_category_scorer.py index 7f73be798..5697d17c5 100644 --- a/pyrit/score/true_false/self_ask_category_scorer.py +++ b/pyrit/score/true_false/self_ask_category_scorer.py @@ -85,7 +85,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_general_true_false_scorer.py b/pyrit/score/true_false/self_ask_general_true_false_scorer.py index 87b70a2bc..edc1cc5c0 100644 --- a/pyrit/score/true_false/self_ask_general_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_general_true_false_scorer.py @@ -92,7 +92,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt_format_string, user_prompt_template=self._prompt_format_string, prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/self_ask_refusal_scorer.py b/pyrit/score/true_false/self_ask_refusal_scorer.py index a2b10c9bb..4341bdc33 100644 --- a/pyrit/score/true_false/self_ask_refusal_scorer.py +++ b/pyrit/score/true_false/self_ask_refusal_scorer.py @@ -79,7 +79,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt_with_objective, prompt_target=self._prompt_target, score_aggregator=self._score_aggregator.__name__, diff --git a/pyrit/score/true_false/self_ask_true_false_scorer.py b/pyrit/score/true_false/self_ask_true_false_scorer.py index 14a213b5a..5097f7be3 100644 --- a/pyrit/score/true_false/self_ask_true_false_scorer.py +++ b/pyrit/score/true_false/self_ask_true_false_scorer.py @@ -155,7 +155,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( system_prompt_template=self._system_prompt, user_prompt_template="objective: {objective}\nresponse: {response}", prompt_target=self._prompt_target, diff --git a/pyrit/score/true_false/substring_scorer.py b/pyrit/score/true_false/substring_scorer.py index 68f79f778..e4548378b 100644 --- a/pyrit/score/true_false/substring_scorer.py +++ b/pyrit/score/true_false/substring_scorer.py @@ -58,7 +58,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ "substring": self._substring, diff --git a/pyrit/score/true_false/true_false_composite_scorer.py b/pyrit/score/true_false/true_false_composite_scorer.py index 17b52f314..5dc2bb818 100644 --- a/pyrit/score/true_false/true_false_composite_scorer.py +++ b/pyrit/score/true_false/true_false_composite_scorer.py @@ -60,7 +60,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=self._scorers, score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/true_false_inverter_scorer.py b/pyrit/score/true_false/true_false_inverter_scorer.py index 243eea43b..85cdd7374 100644 --- a/pyrit/score/true_false/true_false_inverter_scorer.py +++ b/pyrit/score/true_false/true_false_inverter_scorer.py @@ -38,7 +38,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=[self._scorer], score_aggregator=self._score_aggregator.__name__, ) diff --git a/pyrit/score/true_false/video_true_false_scorer.py b/pyrit/score/true_false/video_true_false_scorer.py index 07d543a94..2d50d780e 100644 --- a/pyrit/score/true_false/video_true_false_scorer.py +++ b/pyrit/score/true_false/video_true_false_scorer.py @@ -57,7 +57,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier( + return self._create_identifier( sub_scorers=[self.image_scorer], score_aggregator=self._score_aggregator.__name__, scorer_specific_params={ diff --git a/tests/unit/identifiers/test_converter_identifier.py b/tests/unit/identifiers/test_converter_identifier.py index d1cd56bad..1fef3cb00 100644 --- a/tests/unit/identifiers/test_converter_identifier.py +++ b/tests/unit/identifiers/test_converter_identifier.py @@ -9,8 +9,6 @@ These tests focus on converter-specific fields and behaviors. """ -import pytest - from pyrit.identifiers import ConverterIdentifier @@ -222,4 +220,3 @@ def test_from_dict_with_nested_sub_identifiers(self): assert len(identifier.sub_identifier) == 1 assert isinstance(identifier.sub_identifier[0], ConverterIdentifier) assert identifier.sub_identifier[0].supported_output_types == ("image_path",) - diff --git a/tests/unit/registry/test_scorer_registry.py b/tests/unit/registry/test_scorer_registry.py index 2f29a5886..36421ab00 100644 --- a/tests/unit/registry/test_scorer_registry.py +++ b/tests/unit/registry/test_scorer_registry.py @@ -34,7 +34,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -58,7 +58,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -82,7 +82,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [] diff --git a/tests/unit/score/test_conversation_history_scorer.py b/tests/unit/score/test_conversation_history_scorer.py index 29b956414..0cdbd6cae 100644 --- a/tests/unit/score/test_conversation_history_scorer.py +++ b/tests/unit/score/test_conversation_history_scorer.py @@ -38,7 +38,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -51,7 +51,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] @@ -64,7 +64,7 @@ def __init__(self): super().__init__(validator=ScorerPromptValidator(supported_data_types=["text"])) def _build_identifier(self) -> None: - self._set_identifier() + self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [] diff --git a/tests/unit/score/test_scorer.py b/tests/unit/score/test_scorer.py index e501d3431..5cbb53940 100644 --- a/tests/unit/score/test_scorer.py +++ b/tests/unit/score/test_scorer.py @@ -63,7 +63,7 @@ def __init__(self): def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - return self._set_identifier() + return self._create_identifier() async def _score_async(self, message: Message, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -119,7 +119,7 @@ def __init__(self, *, validator: ScorerPromptValidator): def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this mock scorer.""" - return self._set_identifier() + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: # Track which pieces get scored @@ -1121,7 +1121,7 @@ def __init__(self): def _build_identifier(self) -> ScorerIdentifier: """Build the scorer evaluation identifier for this test scorer.""" - return self._set_identifier() + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None @@ -1307,7 +1307,7 @@ def __init__(self, validator): super().__init__(validator=validator) def _build_identifier(self) -> ScorerIdentifier: - return self._set_identifier() + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_true_false_composite_scorer.py b/tests/unit/score/test_true_false_composite_scorer.py index ac2a4f72c..fc40cd1ce 100644 --- a/tests/unit/score/test_true_false_composite_scorer.py +++ b/tests/unit/score/test_true_false_composite_scorer.py @@ -47,7 +47,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -161,7 +161,7 @@ def __init__(self): self._validator = MagicMock() def _build_identifier(self) -> ScorerIdentifier: - return self._set_identifier() + return self._create_identifier() async def _score_piece_async( self, message_piece: MessagePiece, *, objective: Optional[str] = None diff --git a/tests/unit/score/test_video_scorer.py b/tests/unit/score/test_video_scorer.py index 85a68ce1e..27de6693a 100644 --- a/tests/unit/score/test_video_scorer.py +++ b/tests/unit/score/test_video_scorer.py @@ -75,7 +75,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [ @@ -107,7 +107,7 @@ def _build_identifier(self) -> ScorerIdentifier: Returns: ScorerIdentifier: The identifier for this scorer. """ - return self._set_identifier() + return self._create_identifier() async def _score_piece_async(self, message_piece: MessagePiece, *, objective: Optional[str] = None) -> list[Score]: return [