From d28da2600031a7ebbe74a4c4c4c032cdedce008a Mon Sep 17 00:00:00 2001 From: Kit OC5 Date: Thu, 24 Sep 2026 03:01:59 -0400 Subject: [PATCH] secret-guard: no live credentials in bridged repos (hash-only findings) New guard windy-git/secret-guard (warn-only) over EVERY text file: Telegram, GitHub, AWS, Slack, Anthropic, OpenAI, Stripe live, Google API keys and private-key blocks (scripts/secret_shapes.py, shared with the weekly public scan). A finding carries " #", never the value (house rule 10). Known fakes allowed BY HASH (ci/secret-guard-allow.yml). GUARDS_STATUS gets a secrets column. Leak hunt 09-24: @Windy_0_bot token in a public fixture. Co-Authored-By: Claude Opus 5.5 --- api/tests/test_guards_report.py | 2 +- api/tests/test_secret_guard.py | 84 ++++++++++++++++++++++++ ci/secret-guard-allow.yml | 5 ++ scripts/guards_report.py | 28 ++++---- scripts/pr_status_bridge.py | 7 ++ scripts/secret_guard.py | 110 ++++++++++++++++++++++++++++++++ scripts/secret_shapes.py | 40 ++++++++++++ 7 files changed, 263 insertions(+), 13 deletions(-) create mode 100644 api/tests/test_secret_guard.py create mode 100644 ci/secret-guard-allow.yml create mode 100644 scripts/secret_guard.py create mode 100644 scripts/secret_shapes.py diff --git a/api/tests/test_guards_report.py b/api/tests/test_guards_report.py index 081f635..7a51d09 100644 --- a/api/tests/test_guards_report.py +++ b/api/tests/test_guards_report.py @@ -57,7 +57,7 @@ def test_render_splits_lane_and_grant_counts(): md = gr.render(res) assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md - assert "| windy-git | Windy Git | bbbbbbb | 0 | 0 | clean ✅ |" in md + assert "| windy-git | Windy Git | bbbbbbb | 0 | 0 | 0 | clean ✅ |" in md assert "| windy-pro | Windy Hub | aaaaaaa |" in md # owner = session to message assert "(job reality-check)" in md diff --git a/api/tests/test_secret_guard.py b/api/tests/test_secret_guard.py new file mode 100644 index 0000000..294d4f7 --- /dev/null +++ b/api/tests/test_secret_guard.py @@ -0,0 +1,84 @@ +"""Secret guard: shapes, hash-only findings, allow by hash.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(ROOT / "scripts")) +_spec = importlib.util.spec_from_file_location("secret_guard", ROOT / "scripts" / "secret_guard.py") +sg = importlib.util.module_from_spec(_spec) +sys.modules["secret_guard"] = sg +_spec.loader.exec_module(sg) +ss = sg.ss + +# Synthetic shapes only: none of these is a real credential. +TG = "1234567890:" + "A" * 35 +CASES = [ + ("telegram bot token", f"TELEGRAM_BOT_TOKEN={TG}"), + ("github token", "token = 'ghp_" + "a1" * 18 + "'"), + ("aws access key", "aws_access_key_id = AKIA" + "ABCDEFGHIJKLMNOP"), + ("slack token", "xoxb-" + "1234567890-abcdefghij"), + ("anthropic key", "ANTHROPIC_API_KEY=sk-ant-" + "x" * 30), + ("openai key", "OPENAI_API_KEY=sk-proj-" + "y" * 40), + ("stripe live key", "STRIPE=sk_live_" + "z" * 24), + ("google api key", "key=AIza" + "B" * 35), + ("private key block", "-----BEGIN OPENSSH PRIVATE KEY-----"), +] + + +@pytest.mark.parametrize("kind, text", CASES) +def test_each_shape_is_found_and_only_its_hash_is_kept(kind, text): + hits = sg.scan_line("app.py", text) + assert [k for k, _ in hits] == [kind] + match = hits[0][1] + assert match.startswith(f"{kind} #") and len(match.rsplit("#", 1)[1]) == 8 + # house rule 10: the value itself must never appear in a finding + secret = text.split("=", 1)[-1].strip(" '") + assert secret not in match + + +@pytest.mark.parametrize("text", [ + "sha512-" + "Q" * 86 + "==", # lockfile integrity + "version: 12345678:abc", # short, not a token + "sk-ant-short", # too short + "re_test_register_sends_verification", # windy-pro's fake Resend key + "ANTHROPIC_API_KEY=", # a name, not a value +]) +def test_non_secrets_are_not_flagged(text): + assert sg.scan_line("x", text) == [] + + +def test_anthropic_key_is_not_double_counted_as_openai(): + assert [k for k, _ in sg.scan_line("x", "sk-ant-" + "q" * 40)] == ["anthropic key"] + + +def test_allow_is_by_hash_only(): + F = sg.cg.Finding + fake = F("tests/t.py", 3, "telegram bot token", f"telegram bot token #{ss.h8(TG)}") + real = F("tests/t.py", 9, "telegram bot token", "telegram bot token #deadbeef") + kept = sg._drop_allowed("windy-chat", [fake, real], {"windy-chat": {ss.h8(TG)}}) + assert kept == [real] + assert sg._drop_allowed("windy-mail", [fake], {"windy-chat": {ss.h8(TG)}}) == [fake] + + +def test_allow_file_loads_and_needs_reasons(tmp_path): + assert sg.load_allow() == {} or isinstance(sg.load_allow(), dict) + bad = tmp_path / "a.yml" + bad.write_text("allow:\n - repo: r\n hashes: [abcd1234]\n") + with pytest.raises(ValueError): + sg.load_allow(bad) + + +def test_block_and_warn(monkeypatch): + f = sg.cg.Finding("a.py", 1, "github token", "github token #abcd1234") + monkeypatch.setattr(sg, "MODE", "block") + assert sg.status_for([f], False)[0] == "failure" + assert sg.status_for([], False, grant=[f])[0] == "success" + monkeypatch.setattr(sg, "MODE", "warn") + state, desc, _ = sg.status_for([f], True) + assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 secret-shaped string in tree") diff --git a/ci/secret-guard-allow.yml b/ci/secret-guard-allow.yml new file mode 100644 index 0000000..cf7d18b --- /dev/null +++ b/ci/secret-guard-allow.yml @@ -0,0 +1,5 @@ +# Secret guard allow-list: KNOWN FAKE values that look like secrets (test +# fixtures, docs). Allowed BY HASH (sha256[:8] of the value), never by path, so a +# real secret in the same file still flags. Every entry MUST say why. +# Owner: Windy Git lane (13); changes go through the orchestrator. +allow: [] diff --git a/scripts/guards_report.py b/scripts/guards_report.py index ffa52c5..b02e295 100644 --- a/scripts/guards_report.py +++ b/scripts/guards_report.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Live status of the repo guards (compute-guard + ci-hygiene) as one markdown page. +"""Live status of the repo guards (compute-guard + ci-hygiene + secret-guard) as one markdown page. Scans every bridged repo's DEFAULT branch with both guards and renders what is left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml: @@ -24,6 +24,7 @@ import yaml sys.path.insert(0, str(Path(__file__).resolve().parent)) import ci_hygiene as hy # noqa: E402 import compute_guard as cg # noqa: E402 +import secret_guard as sgd # noqa: E402 ROOT = Path(__file__).resolve().parents[1] OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml")) @@ -101,10 +102,11 @@ def scan(repo: str, owned: list[dict]): return None head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip() sha = cg._git(bare, "rev-parse", head).strip() - out = {"sha": sha, "compute": [], "hygiene": []} + out = {"sha": sha, "compute": [], "hygiene": [], "secrets": []} texts: dict[str, str] = {} for key, fs in (("compute", cg.check(repo, sha, head, True) or []), - ("hygiene", hy.check(repo, sha, head, True) or [])): + ("hygiene", hy.check(repo, sha, head, True) or []), + ("secrets", sgd.check(repo, sha, head, True) or [])): for f in fs: job = None if "/workflows/" in f.path: @@ -117,27 +119,29 @@ def scan(repo: str, owned: list[dict]): def render(results: dict) -> str: now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime()) - lane = {k: 0 for k in ("compute", "hygiene")} - grant = {k: 0 for k in ("compute", "hygiene")} + lane = {k: 0 for k in ("compute", "hygiene", "secrets")} + grant = {k: 0 for k in ("compute", "hygiene", "secrets")} for r in results.values(): for k in lane: - lane[k] += sum(1 for _, _, g in r[k] if not g) - grant[k] += sum(1 for _, _, g in r[k] if g) + lane[k] += sum(1 for _, _, g in r.get(k, []) if not g) + grant[k] += sum(1 for _, _, g in r.get(k, []) if g) L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)", "_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. " "Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "", "| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|", f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |", f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |", - "", "## By repo (lane-owned)", "| Repo | owner | head | compute | hygiene | first items |", "|---|---|---|---|---|---|"] + f"| secret-guard (no credentials in repos; hash only) | {lane['secrets']} | {grant['secrets']} | {'✅ YES' if lane['secrets'] == 0 else '❌ not yet'} |", + "", "## By repo (lane-owned)", "| Repo | owner | head | compute | hygiene | secrets | first items |", "|---|---|---|---|---|---|---|"] for repo, r in sorted(results.items()): c = [x for x in r["compute"] if not x[2]] h = [x for x in r["hygiene"] if not x[2]] - items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (c + h)[:3]) or "clean ✅" - L.append(f"| {repo} | {OWNERS.get(repo, '?')} | {r['sha'][:7]} | {len(c)} | {len(h)} | {items} |") + s = [x for x in r.get("secrets", []) if not x[2]] + items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (s + c + h)[:3]) or "clean ✅" + L.append(f"| {repo} | {OWNERS.get(repo, '?')} | {r['sha'][:7]} | {len(c)} | {len(h)} | {len(s)} | {items} |") L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"] - g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene") - for f, job, own in r[k] if own] + g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene", "secrets") + for f, job, own in r.get(k, []) if own] L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"] return "\n".join(L) + "\n" diff --git a/scripts/pr_status_bridge.py b/scripts/pr_status_bridge.py index a236158..9ec6be0 100755 --- a/scripts/pr_status_bridge.py +++ b/scripts/pr_status_bridge.py @@ -372,6 +372,7 @@ def post_statuses(repo: str, sha: str) -> None: GUARD_CTX = "windy-git/compute-guard" HYGIENE_CTX = "windy-git/ci-hygiene" +SECRET_CTX = "windy-git/secret-guard" def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None: @@ -379,6 +380,11 @@ def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head _post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head) +def post_secret_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None: + """No live credential in a bridged repo (leak hunt 09-24). Findings carry sha256[:8] only.""" + _post_guard("secret_guard", SECRET_CTX, repo, sha, default_branch, is_default_head) + + def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None: """House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only).""" _post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head) @@ -438,6 +444,7 @@ def main() -> int: post_statuses(repo, sha) post_compute_guard(repo, sha, default_branch, sha == default_head) post_ci_hygiene(repo, sha, default_branch, sha == default_head) + post_secret_guard(repo, sha, default_branch, sha == default_head) except Exception as e: # one repo's failure must not hide the others' print(f" FAILED {repo}: {e}") failed = 1 diff --git a/scripts/secret_guard.py b/scripts/secret_guard.py new file mode 100644 index 0000000..cffffb2 --- /dev/null +++ b/scripts/secret_guard.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Secret guard: no live credential lands in a bridged repo (leak hunt 09-24). + +Same walker, cache and GitHub posting as compute_guard / ci_hygiene +(`windy-git/secret-guard`), but over EVERY text file, and a finding carries only +" #", never the value (house rule 10). Known fakes are allowed +BY HASH in ci/secret-guard-allow.yml (repo + hashes + reason). + + sudo python3 scripts/secret_guard.py report [repo ...] +""" + +from __future__ import annotations + +import hashlib +import os +import re +import sys +from pathlib import Path + +import yaml + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import compute_guard as cg # noqa: E402 (shared walker, cache) +import secret_shapes as ss # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] +ALLOW_FILE = Path(os.environ.get("SECRET_GUARD_ALLOW", ROOT / "ci" / "secret-guard-allow.yml")) +MODE = os.environ.get("SECRET_GUARD_MODE", "warn") +NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/") + + +def path_ok(path: str) -> bool: + return not NEVER.search(path) + + +def load_allow(path: Path = ALLOW_FILE) -> dict[str, set[str]]: + """{repo: {hash8, ...}}; every entry needs a reason.""" + data = yaml.safe_load(path.read_text()) if path.exists() else {} + out: dict[str, set[str]] = {} + for e in (data or {}).get("allow") or []: + if not (e.get("repo") and e.get("hashes") and str(e.get("reason", "")).strip()): + raise ValueError(f"allow entry needs repo, hashes and a reason: {e}") + out.setdefault(e["repo"], set()).update(str(h) for h in e["hashes"]) + return out + + +def scan_line(path: str, text: str) -> list[tuple[str, str]]: + return [(kind, f"{kind} #{h}") for kind, h in ss.find(text)] + + +def _drop_allowed(repo: str, findings, allow: dict[str, set[str]]): + ok = allow.get(repo, set()) + return [f for f in findings if f.match.rsplit("#", 1)[-1] not in ok] + + +def check(repo: str, sha: str, default_branch: str, is_default_head: bool): + bare = cg.WORK / f"{repo}.git" + if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle + return None + allow = load_allow() + rules = hashlib.sha256(("|".join(rx.pattern for _, rx in ss.PATTERNS) + ss.PREFILTER).encode()).hexdigest()[:8] + kw = dict(line_fn=scan_line, path_ok=path_ok) + if is_default_head: + fs = cg.cached_scan(f"sec-tree:{repo}:{sha}:{rules}", + lambda: cg.scan_tree(repo, bare, sha, [], prefilter=ss.PREFILTER, **kw)) + else: + fs = cg.cached_scan(f"sec-pr:{repo}:{sha}:{rules}", + lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, [], **kw)) + return _drop_allowed(repo, fs, allow) + + +def status_for(findings, whole_tree: bool, grant=()): + """Same contract as the other guards. `grant` findings never block.""" + scope = "in tree" if whole_tree else "added" + if not findings and grant: + g, n = grant[0], len(grant) + return "success", f"⚠ WARN (Grant-owned, not blocking): {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {g.path}:{g.line} {g.match}"[:140], g + if not findings: + return "success", f"OK: no secret-shaped strings {scope}", None + f, n = findings[0], len(findings) + state = "failure" if MODE == "block" else "success" + lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)" + return state, f"{lead}: {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f + + +def report(repos: list[str]) -> int: + allow = load_allow() + total = 0 + for repo in repos: + bare = cg.WORK / f"{repo}.git" + if not bare.is_dir(): + continue + head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip() + sha = cg._git(bare, "rev-parse", head).strip() + fs = _drop_allowed(repo, cg.scan_tree(repo, bare, sha, [], line_fn=scan_line, path_ok=path_ok, + prefilter=ss.PREFILTER), allow) + total += len(fs) + print(f"## {repo} ({head} {sha[:7]}): {len(fs)} finding(s)") + for f in fs: + print(f" {f.path}:{f.line} {f.match}") + print(f"TOTAL {total}") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) >= 2 and sys.argv[1] == "report": + default = os.environ.get("BRIDGE_REPOS", "").split() or sorted( + p.name.removesuffix(".git") for p in cg.WORK.glob("*.git")) + sys.exit(report(sys.argv[2:] or default)) + sys.exit(__doc__) diff --git a/scripts/secret_shapes.py b/scripts/secret_shapes.py new file mode 100644 index 0000000..6a962c4 --- /dev/null +++ b/scripts/secret_shapes.py @@ -0,0 +1,40 @@ +"""Secret-shaped strings, shared by secret_guard (bridged repos) and +public_secret_scan (weekly, every public repo). A finding NEVER carries the value: +only its kind and sha256[:8] (house rule 10). Leak hunt 09-24: @Windy_0_bot's +token sat in a public repo's test fixture for five months.""" + +from __future__ import annotations + +import hashlib +import re + +# (kind, regex). Order matters only for readability; each match is reported once. +PATTERNS: list[tuple[str, re.Pattern[str]]] = [ + ("telegram bot token", re.compile(r"(? str: + return hashlib.sha256(value.encode() if isinstance(value, str) else value).hexdigest()[:8] + + +def find(text: str) -> list[tuple[str, str]]: + """[(kind, hash8)] for every secret-shaped string in `text`. Values never leave.""" + out = [] + for kind, rx in PATTERNS: + for m in rx.finditer(text): + out.append((kind, h8(m.group(0)))) + return out