{
  "project": "BoundaryLab",
  "generated_at": "2026-10-21T18:18:42.269119+00:00",
  "evidence_type": "Actual portable Python execution on synthetic fixtures",
  "unit_tests": {
    "passed": 18,
    "total": 18,
    "successful": true
  },
  "implementations": [
    {
      "implementation": "vulnerable",
      "passed": 80,
      "total": 165,
      "accepted": false
    },
    {
      "implementation": "reference",
      "passed": 165,
      "total": 165,
      "accepted": true
    }
  ],
  "docker_runtime": "NOT VALIDATED: daemon access denied",
  "real_agents": [
    {
      "id": "codex",
      "name": "Codex",
      "cli_version": "0.163.0",
      "status": "GRADED",
      "models_observed": [],
      "model_disclosure": "CLI default; exact model not emitted in captured events",
      "wall_seconds": 16.032,
      "exit_code": 0,
      "termination": "EXITED",
      "score": {
        "status": "GRADED",
        "passed": 165,
        "total": 165,
        "accepted": true
      },
      "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
      "raw_tool_events_observed": 0,
      "usage_reported": {
        "input_tokens": 11870,
        "cached_input_tokens": 0,
        "cache_write_input_tokens": 0,
        "output_tokens": 533,
        "reasoning_output_tokens": 80
      },
      "response_format_tool_calls": 0,
      "task_tool_events_observed": 0
    },
    {
      "id": "claude",
      "name": "Claude Code",
      "cli_version": "2.1.306",
      "status": "GRADED",
      "models_observed": [
        "claude-opus-5-5"
      ],
      "model_disclosure": "Observed in transcript",
      "wall_seconds": 16.588,
      "exit_code": 0,
      "termination": "EXITED",
      "score": {
        "status": "GRADED",
        "passed": 165,
        "total": 165,
        "accepted": true
      },
      "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
      "raw_tool_events_observed": 1,
      "usage_reported": {
        "input_tokens": 2,
        "cache_creation_input_tokens": 4169,
        "cache_read_input_tokens": 0,
        "output_tokens": 1705,
        "output_tokens_details": {
          "thinking_tokens": 440
        },
        "server_tool_use": {
          "web_search_requests": 0,
          "web_fetch_requests": 0
        },
        "service_tier": "standard",
        "cache_creation": {
          "ephemeral_1h_input_tokens": 4169,
          "ephemeral_5m_input_tokens": 0
        },
        "inference_geo": "global",
        "iterations": [],
        "speed": "standard",
        "fallback_credit": null
      },
      "response_format_tool_calls": 1,
      "task_tool_events_observed": 0
    },
    {
      "id": "agy",
      "name": "Google Antigravity",
      "cli_version": "1.4.4",
      "status": "GRADED",
      "models_observed": [],
      "model_disclosure": "CLI default; exact model not emitted in captured events",
      "wall_seconds": 177.767,
      "exit_code": 0,
      "termination": "EXITED",
      "score": {
        "status": "GRADED",
        "passed": 165,
        "total": 165,
        "accepted": true
      },
      "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
      "raw_tool_events_observed": 1,
      "usage_reported": {
        "input_tokens": 42706,
        "output_tokens": 23699,
        "thinking_tokens": 21964,
        "cache_read_tokens": 0,
        "total_tokens": 66405
      },
      "response_format_tool_calls": 1,
      "task_tool_events_observed": 0
    }
  ],
  "synthetic_traces": "Fabricated teaching examples only; not real-agent evidence",
  "limitations": "Finite educational verifier, public fixture seeds; no certification or proof of general security.",
  "real_cli_pilot": {
    "generated_at": "2026-10-21T18:46:21.675324+00:00",
    "evidence_type": "Real installed CLI code-repair pilot on synthetic fixture data; not synthetic agent traces",
    "protocol": "Identical prompt and starting source; one submitted repair per CLI; no task tools requested; grading in fresh offline non-root namespace",
    "limits": {
      "wall_seconds": 180,
      "cpu_affinity_count": 2,
      "rss_watchdog_bytes": 2147483648,
      "process_watchdog_count": 128,
      "scratch_bytes": 268435456,
      "log_watchdog_bytes": 2097152,
      "max_patch_bytes": 65536
    },
    "limitations": [
      "Default models were used; not all CLIs disclose exact model IDs",
      "RSS/process/log watchdogs are sampled at 100ms, not cgroup hard caps",
      "No common hard token ceiling is claimed; provider usage fields have different semantics",
      "Public five-seed finite verifier only; hidden grading cases were not added",
      "This is code-repair output testing, not a full tool-use or adversarial-agent evaluation",
      "Included subscription usage and disabled overages were confirmed by the user; billing charges were not independently audited"
    ],
    "identical_prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
    "candidates": [
      {
        "id": "codex",
        "name": "Codex",
        "cli_version": "0.163.0",
        "status": "GRADED",
        "models_observed": [],
        "model_disclosure": "CLI default; exact model not emitted in captured events",
        "wall_seconds": 16.032,
        "exit_code": 0,
        "termination": "EXITED",
        "score": {
          "status": "GRADED",
          "passed": 165,
          "total": 165,
          "accepted": true
        },
        "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
        "raw_tool_events_observed": 0,
        "usage_reported": {
          "input_tokens": 11870,
          "cached_input_tokens": 0,
          "cache_write_input_tokens": 0,
          "output_tokens": 533,
          "reasoning_output_tokens": 80
        },
        "response_format_tool_calls": 0,
        "task_tool_events_observed": 0
      },
      {
        "id": "claude",
        "name": "Claude Code",
        "cli_version": "2.1.306",
        "status": "GRADED",
        "models_observed": [
          "claude-opus-5-5"
        ],
        "model_disclosure": "Observed in transcript",
        "wall_seconds": 16.588,
        "exit_code": 0,
        "termination": "EXITED",
        "score": {
          "status": "GRADED",
          "passed": 165,
          "total": 165,
          "accepted": true
        },
        "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
        "raw_tool_events_observed": 1,
        "usage_reported": {
          "input_tokens": 2,
          "cache_creation_input_tokens": 4169,
          "cache_read_input_tokens": 0,
          "output_tokens": 1705,
          "output_tokens_details": {
            "thinking_tokens": 440
          },
          "server_tool_use": {
            "web_search_requests": 0,
            "web_fetch_requests": 0
          },
          "service_tier": "standard",
          "cache_creation": {
            "ephemeral_1h_input_tokens": 4169,
            "ephemeral_5m_input_tokens": 0
          },
          "inference_geo": "global",
          "iterations": [],
          "speed": "standard",
          "fallback_credit": null
        },
        "response_format_tool_calls": 1,
        "task_tool_events_observed": 0
      },
      {
        "id": "agy",
        "name": "Google Antigravity",
        "cli_version": "1.4.4",
        "status": "GRADED",
        "models_observed": [],
        "model_disclosure": "CLI default; exact model not emitted in captured events",
        "wall_seconds": 177.767,
        "exit_code": 0,
        "termination": "EXITED",
        "score": {
          "status": "GRADED",
          "passed": 165,
          "total": 165,
          "accepted": true
        },
        "prompt_sha256": "be2932fd3d6b76d9770c15e2950da4798ce05eab468c3138d78db2546409d980",
        "raw_tool_events_observed": 1,
        "usage_reported": {
          "input_tokens": 42706,
          "output_tokens": 23699,
          "thinking_tokens": 21964,
          "cache_read_tokens": 0,
          "total_tokens": 66405
        },
        "response_format_tool_calls": 1,
        "task_tool_events_observed": 0
      }
    ]
  }
}