{
  "@context": "https://schema.org",
  "@type": "Dataset",
  "name": "Contemporary Agent Attacks: a reproducible prompt-injection and agent-attack benchmark",
  "alternateName": "contemporary-agent-attacks",
  "description": "A public, versioned, clone-and-run benchmark of 497 real AI-agent attacks (395 open plus 102 held-out) across 22 attack categories, paired with 1,172 benign samples, built to measure prompt-injection and agent-attack detection. Measured on this corpus, Crawdad's full detection engine detects 496 of 497 attacks (99.80%) at a single false positive out of 1,172 (0.09%); with the ML layer disabled the deterministic pattern-only floor is 188 of 497 (37.8%). The benchmark names its own errors: the one missed attack is a bare social-engineering pretext opener in the human_agent_trust held-out category, and the one false positive is a Stack Overflow question about Go method-receiver syntax that includes GitHub source links, flagged by the pattern layers rather than the ML classifier. Both are published in the corpus. Last full re-verification 2026-08-02. This is an evaluation corpus, not training data. No independent third party has re-run it against a running sidecar.",
  "url": "https://getcrawdad.dev/trust#benchmark",
  "sameAs": "https://github.com/AndrewSispoidis/contemporary-agent-attacks",
  "identifier": "contemporary-agent-attacks",
  "version": "1.0.0",
  "inLanguage": "en",
  "isAccessibleForFree": true,
  "license": "https://creativecommons.org/licenses/by/4.0/",
  "creditText": "Crawdad Security, Inc.",
  "creator": {
    "@type": "Organization",
    "name": "Crawdad Security, Inc.",
    "url": "https://getcrawdad.dev"
  },
  "publisher": {
    "@type": "Organization",
    "name": "Crawdad Security, Inc.",
    "url": "https://getcrawdad.dev"
  },
  "dateModified": "2026-08-02",
  "measurementTechnique": "Each sample is passed through the full detection engine (deterministic pattern layers plus a custom-trained DeBERTa-small classifier run on every input) using crawdad-corpus-verify --require-ml, or externally against a running sidecar via python3 benchmark/run.py. Attacks are scored detected or missed; benign samples are scored clean or false positive.",
  "keywords": [
    "prompt injection benchmark",
    "AI agent security benchmark",
    "indirect prompt injection",
    "agent attack detection",
    "reproducible benchmark",
    "benchmark with named misses",
    "cascading_failures",
    "credential_exfiltration_input",
    "credential_exfiltration_output",
    "data_exfiltration_intent",
    "direct_prompt_injection",
    "encoding_obfuscation",
    "goal_hijack",
    "human_agent_trust",
    "indirect_prompt_injection",
    "inter_agent_communication",
    "jailbreak",
    "memory_poisoning",
    "misinformation",
    "pii_exposure",
    "rogue_agents",
    "supply_chain_mcp",
    "supply_chain_text",
    "system_prompt_leakage",
    "tool_abuse_actual",
    "tool_abuse_english",
    "unbounded_consumption",
    "vector_embedding"
  ],
  "variableMeasured": [
    {
      "@type": "PropertyValue",
      "name": "Detection rate (full engine, ML active)",
      "value": "99.80%",
      "description": "496 of 497 attacks detected with the ML classifier run on every input."
    },
    {
      "@type": "PropertyValue",
      "name": "False-positive rate",
      "value": "0.09%",
      "description": "1 of 1,172 benign samples flagged."
    },
    {
      "@type": "PropertyValue",
      "name": "Pattern-only detection floor (ML disabled)",
      "value": "37.8%",
      "description": "188 of 497 attacks detected by the deterministic pattern layers alone; the ML layer is a booster on top, not a gate."
    },
    {
      "@type": "PropertyValue",
      "name": "F1 score",
      "value": "99.80%"
    },
    {
      "@type": "PropertyValue",
      "name": "Attacks",
      "value": 497,
      "description": "395 open plus 102 held-out."
    },
    {
      "@type": "PropertyValue",
      "name": "Benign samples",
      "value": 1172
    },
    {
      "@type": "PropertyValue",
      "name": "Attack categories",
      "value": 22
    }
  ],
  "distribution": [
    {
      "@type": "DataDownload",
      "name": "Corpus, methodology, and reproducible runner (source repository)",
      "encodingFormat": "application/json",
      "contentUrl": "https://github.com/AndrewSispoidis/contemporary-agent-attacks"
    },
    {
      "@type": "DataDownload",
      "name": "Machine-readable dataset record (this document)",
      "encodingFormat": "application/ld+json",
      "contentUrl": "https://getcrawdad.dev/benchmark-dataset.json"
    }
  ],
  "citation": "Crawdad Security, Inc. Contemporary Agent Attacks: a reproducible prompt-injection benchmark. CC BY 4.0. https://github.com/AndrewSispoidis/contemporary-agent-attacks"
}