Giskard

pip install --prerelease allow "giskard[scan,openai]>=3.0.0b0" openai "pydantic==2.12.5"

export ABLIT_KEY="ak_YOUR_API_KEY"

import os

import giskard.llm
from giskard.agents import Generator
from giskard.checks import set_default_generator

giskard.llm.configure(
    "openai",  # the built-in provider alias Giskard components fall back to
    provider="openai",
    api_key=os.environ["ABLIT_KEY"],
    base_url="https://api.abliteration.ai/v1",
    timeout=120.0,
)
set_default_generator(Generator(model="openai/abliterated-model"))

from giskard.checks import Trace

async def my_agent(inputs: str, trace: Trace) -> str:
    messages = [{"role": "system", "content": SYSTEM_PROMPT}]
    for interaction in trace.interactions:
        messages.append({"role": "user", "content": interaction.inputs})
        messages.append({"role": "assistant", "content": interaction.outputs})
    messages.append({"role": "user", "content": inputs})

    # ... run your agent (any framework, any model, with or without tools)
    # and return its final reply
    return reply

from giskard.scan import vulnerability_scan

result = await vulnerability_scan(
    target=my_agent,
    description=(
        "A customer support agent for Acme Store that can look up order "
        "status, issue refunds, and look up customer profiles via tools."
    ),
    languages=["en"],
    target_mode="multiturn",  # attack the agent across multiple turns
    max_scenarios=6,          # keep the first run small
    seed=42,
    parallel=True,
    max_concurrency=2,        # be gentle with the endpoint
    return_exception=True,    # keep scanning if a probe errors
)

from giskard.scan.vulnerability import vulnerability_suite_generator_registry
from giskard.scan.generators.prompt_injection import PromptInjectionScenarioGenerator
from giskard.scan.generators.adversarial import AdversarialScenarioGenerator

vulnerability_suite_generator_registry.clear()
vulnerability_suite_generator_registry.register(PromptInjectionScenarioGenerator)  # bundled OWASP LLM01 probes
vulnerability_suite_generator_registry.register(
    AdversarialScenarioGenerator(max_turns=2)  # LLM-written attacks + judge
)

result.print_report(group_by="threat-type")

from pathlib import Path
Path("report.json").write_text(result.model_dump_json(indent=2))