From b848d82b586f5507bca6dc1f246b6e088218694a Mon Sep 17 00:00:00 2001 From: Nicolas MASSE Date: Sun, 11 Oct 2026 12:58:54 +0200 Subject: [PATCH] Add redacted PR #47 subjects and show results as colored dots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bench svn-arv/redacted#47 (Rust rewrite) as is and with the entropy heuristic on. Known values are now escaped instead of using \Q...\E, which Rust's regex lacks. Subject files may live in subdirectories. Matrix cells become ๐Ÿ”ด (sent to the API), ๐ŸŸ  (transcript only) or ๐ŸŸข. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01GZY2T82aEHnuVC4ALzRcPh --- README.md | 70 ++++++++++++++++++++++++++++++++------------- bench/runner.py | 1 + bench/subjects.py | 60 +++++++++++++++++++++++++++++++------- conftest.py | 6 ++-- tests/test_leaks.py | 5 ++-- 5 files changed, 107 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 3be4800..b28444f 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,20 @@ Each test: 2. runs `claude -p` behind a **local proxy that records everything sent to the API** (`ANTHROPIC_BASE_URL`); 3. checks in the `stream-json` output that the model really performed the - requested action. If not, the result is **inconclusive** (skip), never "clean"; + requested action. If not, the result is **inconclusive** (skip), never ๐ŸŸข; 4. looks for the canary in three places: -| Letter | Where | What it proves | -|---|---|---| -| `w` | traffic to the API | the secret was sent to Anthropic: this is the reference | -| `t` | transcript `~/.claude/projects/โ€ฆ/.jsonl` | the secret was written to disk | -| `a` | the model's final answer | informative only | +| Where | What it proves | +|---|---| +| traffic to the API | the secret was sent to Anthropic: this is the reference | +| transcript `~/.claude/projects/โ€ฆ/.jsonl` | the secret was written to disk | +| the model's final answer | informative only (`results.json`) | + +Each cell of the matrix shows the worst place reached: + +- ๐Ÿ”ด the secret was sent to the Anthropic API; +- ๐ŸŸ  it was not sent to the API, but was written to the transcript on disk; +- ๐ŸŸข neither. In every mode, Claude Code runs with `--setting-sources project` (your global hooks are ignored), test-specific `--settings` and `--strict-mcp-config`. @@ -27,18 +33,24 @@ hooks are ignored), test-specific `--settings` and `--strict-mcp-config`. ## Results Run on 2026-10-11, podman sandbox, Claude Code 2.1.283, model Haiku 4.5, -`redacted` at commit `0b4dd23` (`main`, Go version). - -| Case | bare | redacted | -|---|---|---| -| `cat_file` | LEAK(w,t,a) | LEAK(w,t) | -| `read_file` | LEAK(w,t,a) | LEAK(w,t) | -| `env` | LEAK(w,t) | LEAK(w,t) | -| `unlabeled_random` | LEAK(w,t,a) | LEAK(w,t) | -| `known_value` | LEAK(w,t) | LEAK(w,t) | -| `tail_foreground` | LEAK(w,t,a) | LEAK(w,t) | -| `tail_background` | LEAK(w,t,a) | LEAK(w,t) | -| `monitor` | LEAK(w,t,a) | LEAK(w,t) | +`redacted` at commit `0b4dd23` (`main`, Go version), `redacted-pr47` at +commit `b88d507` ([svn-arv/redacted#47](https://github.com/svn-arv/redacted/pull/47), +Rust rewrite 1.0.0), as is and with the entropy heuristic on +(`redacted-pr47-heuristic`). + +| Case | bare | redacted | redacted-pr47 | redacted-pr47-heuristic | +|---|---|---|---|---| +| `cat_file` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `read_file` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `env` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `unlabeled_random` | ๐Ÿ”ด | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | +| `known_value` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `tail_foreground` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `tail_background` | ๐Ÿ”ด | ๐Ÿ”ด | ๐ŸŸข | ๐ŸŸข | +| `monitor` | ๐Ÿ”ด | ๐Ÿ”ด | ๐Ÿ”ด | ๐Ÿ”ด | + +๐Ÿ”ด sent to the Anthropic API ยท ๐ŸŸ  not sent to the API, but written to the +transcript on disk ยท ๐ŸŸข neither - **bare**: the canary leaks in every case, as expected. This confirms the bench detects leaks. @@ -47,8 +59,17 @@ Run on 2026-10-11, podman sandbox, Claude Code 2.1.283, model Haiku 4.5, answers with `decision: "block"`, which does not replace the tool output: the raw output goes to the model and the redacted copy is only added beside it (see [svn-arv/redacted#46](https://github.com/svn-arv/redacted/issues/46)). - The `a` disappears only because the model, seeing the warning, no longer - repeats the secret. + The model, seeing the warning, merely stops repeating the secret in its + answer. +- **redacted-pr47**: `updatedToolOutput` now mirrors the shape of the tool + response, so Claude Code applies the replacement: the model and the + transcript only see the marker (`DB_PASSWORD= [REDACTED ...grtZ]`, last 4 + characters kept). `unlabeled_random` leaks because nothing flags a random + value under a neutral key: in 1.0.0 that takes a secret learned by + `redacted init` (global config, absent from the sandbox) or the entropy + heuristic (off by default). +- **redacted-pr47-heuristic**: with the heuristic on, `unlabeled_random` is + caught too (`cache_ref: [REDACTED ...B95I]`, pattern `secret_value`). - **`monitor`**: `Monitor` events reach the model as notifications that never go through PostToolUse hooks, so a PostToolUse filter cannot see them at all. - **`tail_foreground`**: when the Bash timeout expires, Claude Code moves the @@ -122,6 +143,14 @@ one. - `redacted`: [svn-arv/redacted](https://github.com/svn-arv/redacted) as a PostToolUse hook. Binary from `REDACTED_BIN` or the `PATH`. The test passes when nothing leaks (neither `w` nor `t`). +- `redacted-pr47`: the same, built from + [svn-arv/redacted#47](https://github.com/svn-arv/redacted/pull/47) (Rust + rewrite). Binary from `REDACTED_PR47_BIN` only. The sandbox has no global + `~/.config/redacted`, so it runs without learned secrets or heuristic: + vendor signatures and project `.redacted.engine.yml` patterns only. +- `redacted-pr47-heuristic`: the same binary with `heuristic.enabled: true`. + The heuristic is read from the global config only, so the hook runs with a + `HOME` of its own holding `.config/redacted/config.yaml`. To test another project, add a `Subject` subclass that provides `files()` (binaries copied into the sandbox), `settings(where)` (the hooks, given each @@ -135,6 +164,7 @@ register it in `SUBJECTS`. A hook binary must be able to run on Fedora minimal uv run pytest # everything uv run pytest --subject bare --case monitor # a single case REDACTED_BIN=~/git/redacted/redacted uv run pytest --subject redacted +REDACTED_PR47_BIN=~/git/redacted-pr47/target/release/redacted uv run pytest --subject redacted-pr47 --subject redacted-pr47-heuristic uv run pytest --model claude-sonnet-5 # another model (default: Haiku 4.5) uv run pytest --claude-version 2.1.292 # another Claude Code version (podman) uv run pytest --sandbox host # no container diff --git a/bench/runner.py b/bench/runner.py index 59fdac7..44f8f52 100644 --- a/bench/runner.py +++ b/bench/runner.py @@ -220,6 +220,7 @@ class PodmanSandbox: d.mkdir() (config / ".claude.json").write_text(json.dumps(self.feature_cache)) for name, src in subject_files.items(): + (staged / name).parent.mkdir(parents=True, exist_ok=True) shutil.copy2(src, staged / name) (staged / "settings.json").write_text(json.dumps(settings, indent=2)) shutil.copy2(staged / "settings.json", artifacts / "settings.json") diff --git a/bench/subjects.py b/bench/subjects.py index 0adb5e3..c97688c 100644 --- a/bench/subjects.py +++ b/bench/subjects.py @@ -7,13 +7,16 @@ register it in SUBJECTS. Files are copied into the sandbox: `settings(where)` receives, for each `files()` name, the path the file is visible at (a container path under podman). +A name may contain `/` to lay out a directory tree. Hook binaries must run in the sandbox image (Fedora minimal, glibc). """ from __future__ import annotations import os +import re import shutil +import tempfile from dataclasses import dataclass from pathlib import Path @@ -49,20 +52,23 @@ class Bare(Subject): class Redacted(Subject): - """github.com/svn-arv/redacted (Go, 0.x): a PostToolUse hook, `redacted scrub`. + """github.com/svn-arv/redacted: a PostToolUse hook, `redacted scrub`. - Known values are declared as a literal custom pattern in the project's - .redacted.engine.yml. In host mode the global ~/.config/redacted files - also apply; the podman sandbox has none. + `redacted` is the Go version (0.x, `main`), `redacted-pr47` the Rust + rewrite of svn-arv/redacted#47 (1.0.0). Known values are declared as a + literal custom pattern in the project's .redacted.engine.yml, which both + versions read. In host mode the global ~/.config/redacted files also + apply; the podman sandbox has none, so 1.0.0 runs without learned secrets. """ - def __init__(self, name: str = "redacted", binary: str | None = None) -> None: + def __init__(self, name: str = "redacted", env_var: str = "REDACTED_BIN", which: bool = True) -> None: super().__init__(name) - self.binary = binary or os.environ.get("REDACTED_BIN") or shutil.which("redacted") or "" + self.env_var = env_var + self.binary = os.environ.get(env_var) or (which and shutil.which("redacted")) or "" def available(self) -> str | None: if not self.binary or not Path(self.binary).exists(): - return "redacted binary not found (set REDACTED_BIN)" + return f"{self.name} binary not found (set {self.env_var})" return None def files(self) -> dict[str, Path]: @@ -73,10 +79,44 @@ class Redacted(Subject): return {"hooks": {"PostToolUse": [{"hooks": [{"type": "command", "command": cmd}]}]}} def declare_known(self, project: Path, values: list[str]) -> bool: - assert not any("\\E" in v or "'" in v for v in values), "value cannot be quoted literally" - regex = "|".join(r"\Q" + v + r"\E" for v in values) + # No \Q...\E: Rust's regex (1.0.0) lacks it. Escaping punctuation works in both. + assert not any("'" in v or re.search(r"\s", v) for v in values), "value cannot be quoted literally" + regex = "|".join(re.sub(r"([^0-9A-Za-z_])", r"\\\1", v) for v in values) (project / ".redacted.engine.yml").write_text(f"patterns:\n - name: bench_known\n regex: '{regex}'\n") return True -SUBJECTS: dict[str, Subject] = {s.name: s for s in [Bare(), Redacted()]} +class RedactedHeuristic(Redacted): + """redacted 1.0.0 with the entropy heuristic on (`heuristic.enabled: true`). + + It reads the heuristic from the global config only, so the hook runs with a + HOME of its own that holds .config/redacted/config.yaml. Stats still go to + the real HOME. + """ + + CONFIG = "home/.config/redacted/config.yaml" + + def __init__(self) -> None: + super().__init__("redacted-pr47-heuristic", env_var="REDACTED_PR47_BIN", which=False) + self._config: Path | None = None + + def files(self) -> dict[str, Path]: + if self._config is None: + self._config = Path(tempfile.mkdtemp(prefix="leak-bench-redacted-")) / ".config/redacted/config.yaml" + self._config.parent.mkdir(parents=True) + self._config.write_text("version: 2\nheuristic:\n enabled: true\n") + return {**super().files(), self.CONFIG: self._config} + + def settings(self, where: dict[str, str]) -> dict: + home = where[self.CONFIG].removesuffix("/.config/redacted/config.yaml") + cmd = (f'REDACTED_STATS_FILE="$HOME/.config/redacted/stats.jsonl" HOME={home} ' + f"{where['redacted']} scrub") + return {"hooks": {"PostToolUse": [{"hooks": [{"type": "command", "command": cmd}]}]}} + + +SUBJECTS: dict[str, Subject] = {s.name: s for s in [ + Bare(), + Redacted(), + Redacted("redacted-pr47", env_var="REDACTED_PR47_BIN", which=False), + RedactedHeuristic(), +]} diff --git a/conftest.py b/conftest.py index 6ce7157..5d9bd74 100644 --- a/conftest.py +++ b/conftest.py @@ -93,9 +93,9 @@ def pytest_terminal_summary(terminalreporter, exitstatus, config) -> None: lines = ["| case | " + " | ".join(subjects) + " |", "|---" * (len(subjects) + 1) + "|"] for c in cases: lines.append(f"| {c} | " + " | ".join(cell.get((s, c), "") for s in subjects) + " |") - legend = ("LEAK(w,t,a) = canary found on the wire (sent to the API), in the transcript on disk, " - "in the final answer. clean = found nowhere. ? = inconclusive (scenario not exercised). " - "n/a = subject cannot declare known values.") + legend = ("๐Ÿ”ด = canary sent to the Anthropic API. ๐ŸŸ  = not sent to the API, but written to the " + "transcript on disk. ๐ŸŸข = neither. ? = inconclusive (scenario not exercised). " + "n/a = subject cannot declare known values. skip = subject not available.") terminalreporter.write_sep("=", "leak matrix") for line in lines: diff --git a/tests/test_leaks.py b/tests/test_leaks.py index 05da1bc..4893758 100644 --- a/tests/test_leaks.py +++ b/tests/test_leaks.py @@ -9,8 +9,9 @@ from bench.runner import run_claude def _cell(leaks: dict[str, bool]) -> str: - where = ",".join(k[0] for k, v in leaks.items() if v) - return f"LEAK({where})" if where else "clean" + if leaks["wire"]: + return "๐Ÿ”ด" + return "๐ŸŸ " if leaks["transcript"] else "๐ŸŸข" def test_leak(subject, case_name, tmp_path, request, report_dir, record, sandbox):