Compare commits
21 Commits
6440c7d5f4
...
veron-olla
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd0400c371 | ||
|
|
4e7ab667ed | ||
|
|
f10db19316 | ||
|
|
3503894a1e | ||
|
|
fbab5c4669 | ||
|
|
1b552c0882 | ||
|
|
8ebc18c3bc | ||
|
|
9566826ce9 | ||
|
|
160a3d69ba | ||
|
|
7e28a23c22 | ||
|
|
1da4d39c0b | ||
|
|
53fbcfe78f | ||
|
|
1c552e94de | ||
|
|
8690c4f8d3 | ||
|
|
313f41b49c | ||
|
|
ea02ade0cb | ||
|
|
497222094f | ||
|
|
d28da26000 | ||
|
|
04c857654a | ||
|
|
32512a32fc | ||
|
|
b52e6b4784 |
@@ -229,3 +229,24 @@ def test_block_mode_never_blocks_grant_owned(monkeypatch):
|
|||||||
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
||||||
lane = cg.Finding("a.py", 3, "provider host", "x")
|
lane = cg.Finding("a.py", 3, "provider host", "x")
|
||||||
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
||||||
|
|
||||||
|
|
||||||
|
def test_veron_ollama_warns_on_added_lines_and_never_blocks(monkeypatch):
|
||||||
|
hits = cg.scan_line("app/llm.py", 'OLLAMA = "http://192.168.1.73:11434/api/generate"')
|
||||||
|
assert [k for k, _ in hits] == ["veron ollama"]
|
||||||
|
assert cg.scan_line("app/llm.py", 'port = 114345') == [] # not the port
|
||||||
|
assert cg.scan_line("app/llm.py", "# was http://x:11434 (removed)") == [] # a comment is not a call
|
||||||
|
f = cg.Finding("app/llm.py", 7, "veron ollama", ":11434")
|
||||||
|
monkeypatch.setattr(cg, "MODE", "block")
|
||||||
|
state, desc, _ = cg.status_for([f], whole_tree=False)
|
||||||
|
assert state == "success" and desc.startswith("⚠ WARN: new Veron Ollama ref app/llm.py:7")
|
||||||
|
assert "Windy Mind" in desc and len(desc) <= 140
|
||||||
|
hard = cg.Finding("app/llm.py", 1, "provider host", "api.openai.com")
|
||||||
|
assert cg.status_for([f, hard], whole_tree=False)[0] == "failure" # a real violation still blocks
|
||||||
|
|
||||||
|
|
||||||
|
def test_ollama_in_added_pr_lines_only():
|
||||||
|
diff = ("+++ b/svc/client.py\n@@ -0,0 +1,2 @@\n+import httpx\n"
|
||||||
|
"+URL = 'http://veron:11434/api/chat'\n")
|
||||||
|
got = cg.parse_added("some-repo", diff, [])
|
||||||
|
assert [(f.kind, f.line) for f in got] == [("veron ollama", 2)]
|
||||||
|
|||||||
@@ -57,7 +57,7 @@ def test_render_splits_lane_and_grant_counts():
|
|||||||
md = gr.render(res)
|
md = gr.render(res)
|
||||||
assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md
|
assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md
|
||||||
assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md
|
assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md
|
||||||
assert "| windy-git | Windy Git | bbbbbbb | 0 | 0 | clean ✅ |" in md
|
assert "| windy-git | Windy Git | bbbbbbb | 0 | 0 | 0 | clean ✅ |" in md
|
||||||
assert "| windy-pro | Windy Hub | aaaaaaa |" in md # owner = session to message
|
assert "| windy-pro | Windy Hub | aaaaaaa |" in md # owner = session to message
|
||||||
assert "(job reality-check)" in md
|
assert "(job reality-check)" in md
|
||||||
|
|
||||||
|
|||||||
77
api/tests/test_lockbox_put.py
Normal file
77
api/tests/test_lockbox_put.py
Normal file
@@ -0,0 +1,77 @@
|
|||||||
|
"""lockbox-put against a LOCAL fake lockbox repo only (never the real one)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
FAKE = "Zk3vQ8mT1pLw7Xn2Rb5Hd9Yc" # synthetic
|
||||||
|
|
||||||
|
|
||||||
|
def sh(*a, cwd=None):
|
||||||
|
return subprocess.run(a, cwd=cwd, check=True, capture_output=True, text=True)
|
||||||
|
|
||||||
|
|
||||||
|
def make_remote(tmp_path):
|
||||||
|
work = tmp_path / "seed"
|
||||||
|
work.mkdir()
|
||||||
|
sh("git", "init", "-q", "-b", "main", cwd=work)
|
||||||
|
(work / "ACCESS_LOCKBOX.md").write_text("# LOCKBOX\n\n- **`EXISTING_KEY`**: `abcdefgh12345678`\nOLD_ENV_KEY=whatever123456\n")
|
||||||
|
sh("git", "add", "-A", cwd=work)
|
||||||
|
sh("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "seed", cwd=work)
|
||||||
|
bare = tmp_path / "remote.git"
|
||||||
|
sh("git", "clone", "-q", "--bare", str(work), str(bare))
|
||||||
|
return bare
|
||||||
|
|
||||||
|
|
||||||
|
def put(tmp_path, bare, key, content, mode=0o600):
|
||||||
|
f = tmp_path / "val"
|
||||||
|
f.write_text(content)
|
||||||
|
os.chmod(f, mode)
|
||||||
|
e = {**os.environ, "LOCKBOX_PUT_REPO": str(bare), "LOCKBOX_PUT_NO_PR": "1", "HOME": str(tmp_path / "h")}
|
||||||
|
(tmp_path / "h" / ".cache").mkdir(parents=True, exist_ok=True)
|
||||||
|
r = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_put.py"), key, str(f), "--lane", "test", "--note", "n"],
|
||||||
|
capture_output=True, text=True, env=e)
|
||||||
|
return r.returncode, r.stdout + r.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_appends_one_key_by_branch_and_never_echoes_value(tmp_path):
|
||||||
|
bare = make_remote(tmp_path)
|
||||||
|
rc, out = put(tmp_path, bare, "NEW_TEST_KEY", FAKE)
|
||||||
|
assert rc == 0 and "pushed branch lockbox-put/new_test_key-" in out
|
||||||
|
assert FAKE not in out and FAKE[:6] not in out
|
||||||
|
br = [b.strip() for b in sh("git", "branch", "--list", "lockbox-put/*", cwd=bare).stdout.splitlines()]
|
||||||
|
assert len(br) == 1
|
||||||
|
diff = sh("git", "diff", "--numstat", f"main..{br[0]}", cwd=bare).stdout.split()
|
||||||
|
assert diff[1] == "0" and diff[2] == "ACCESS_LOCKBOX.md" # additions only
|
||||||
|
content = sh("git", "show", f"{br[0]}:ACCESS_LOCKBOX.md", cwd=bare).stdout
|
||||||
|
assert f"- **`NEW_TEST_KEY`**: `{FAKE}`" in content and "EXISTING_KEY" in content
|
||||||
|
# main is untouched
|
||||||
|
assert FAKE not in sh("git", "show", "main:ACCESS_LOCKBOX.md", cwd=bare).stdout
|
||||||
|
|
||||||
|
|
||||||
|
def test_refuses_existing_key_both_formats_and_bad_input(tmp_path):
|
||||||
|
bare = make_remote(tmp_path)
|
||||||
|
for k in ("EXISTING_KEY", "OLD_ENV_KEY"):
|
||||||
|
rc, out = put(tmp_path, bare, k, FAKE)
|
||||||
|
assert rc == 3 and "already exists" in out and FAKE not in out
|
||||||
|
assert put(tmp_path, bare, "lower_case", FAKE)[0] == 2
|
||||||
|
assert put(tmp_path, bare, "OK_KEY_1", FAKE, mode=0o644)[0] == 2 # not 0600
|
||||||
|
assert put(tmp_path, bare, "OK_KEY_2", "has space `tick`")[0] == 2 # unsafe value
|
||||||
|
assert not sh("git", "branch", "--list", "lockbox-put/*", cwd=bare).stdout.strip() # nothing pushed
|
||||||
|
|
||||||
|
|
||||||
|
def test_symlink_refused(tmp_path):
|
||||||
|
bare = make_remote(tmp_path)
|
||||||
|
real = tmp_path / "real"
|
||||||
|
real.write_text(FAKE)
|
||||||
|
os.chmod(real, 0o600)
|
||||||
|
link = tmp_path / "link"
|
||||||
|
link.symlink_to(real)
|
||||||
|
e = {**os.environ, "LOCKBOX_PUT_REPO": str(bare), "LOCKBOX_PUT_NO_PR": "1"}
|
||||||
|
r = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_put.py"), "SYM_KEY", str(link)],
|
||||||
|
capture_output=True, text=True, env=e)
|
||||||
|
assert r.returncode == 2 and FAKE not in r.stdout + r.stderr
|
||||||
@@ -451,3 +451,13 @@ def test_failed_grant_split_warns_instead_of_blocking(fake, monkeypatch):
|
|||||||
monkeypatch.setitem(sys.modules, "guards_report", type("GR", (), {"split_grant": staticmethod(boom)}))
|
monkeypatch.setitem(sys.modules, "guards_report", type("GR", (), {"split_grant": staticmethod(boom)}))
|
||||||
bridge.post_compute_guard("windy-pro", SHA, "main", True)
|
bridge.post_compute_guard("windy-pro", SHA, "main", True)
|
||||||
assert [p["state"] for p in f.posted] == ["success"]
|
assert [p["state"] for p in f.posted] == ["success"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("name, hidden", [
|
||||||
|
("Docker Build", True), ("docker-build", True), ("Docker build", True), ("docker", True),
|
||||||
|
("Boot smoke (no Docker)", False), ("smoke (no-docker)", False), ("boot without Docker", False),
|
||||||
|
("smoke", False),
|
||||||
|
])
|
||||||
|
def test_no_docker_smoke_jobs_are_posted(name, hidden):
|
||||||
|
"""windy-search #96: its replacement job says "no Docker" and was hidden."""
|
||||||
|
assert bridge.needs_daemon("windy-search", "ci", name) is hidden
|
||||||
|
|||||||
156
api/tests/test_secret_guard.py
Normal file
156
api/tests/test_secret_guard.py
Normal file
@@ -0,0 +1,156 @@
|
|||||||
|
"""Secret guard: shapes, hash-only findings, allow by hash."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / "scripts"))
|
||||||
|
_spec = importlib.util.spec_from_file_location("secret_guard", ROOT / "scripts" / "secret_guard.py")
|
||||||
|
sg = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["secret_guard"] = sg
|
||||||
|
_spec.loader.exec_module(sg)
|
||||||
|
ss = sg.ss
|
||||||
|
|
||||||
|
# Synthetic shapes only: none of these is a real credential.
|
||||||
|
TG = "1234567890:" + "A" * 35
|
||||||
|
CASES = [
|
||||||
|
("telegram bot token", f"TELEGRAM_BOT_TOKEN={TG}"),
|
||||||
|
("github token", "token = 'ghp_" + "a1" * 18 + "'"),
|
||||||
|
("aws access key", "aws_access_key_id = AKIA" + "ABCDEFGHIJKLMNOP"),
|
||||||
|
("slack token", "xoxb-" + "1234567890-abcdefghij"),
|
||||||
|
("anthropic key", "ANTHROPIC_API_KEY=sk-ant-" + "x" * 30),
|
||||||
|
("openai key", "OPENAI_API_KEY=sk-proj-" + "y" * 40),
|
||||||
|
("stripe live key", "STRIPE=sk_live_" + "z" * 24),
|
||||||
|
("google api key", "key=AIza" + "B" * 35),
|
||||||
|
("private key block", "-----BEGIN OPENSSH PRIVATE KEY-----"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("kind, text", CASES)
|
||||||
|
def test_each_shape_is_found_and_only_its_hash_is_kept(kind, text):
|
||||||
|
hits = sg.scan_line("app.py", text)
|
||||||
|
assert [k for k, _ in hits] == [kind]
|
||||||
|
match = hits[0][1]
|
||||||
|
assert match.startswith(f"{kind} #") and len(match.rsplit("#", 1)[1]) == 8
|
||||||
|
# house rule 10: the value itself must never appear in a finding
|
||||||
|
secret = text.split("=", 1)[-1].strip(" '")
|
||||||
|
assert secret not in match
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text", [
|
||||||
|
"sha512-" + "Q" * 86 + "==", # lockfile integrity
|
||||||
|
"version: 12345678:abc", # short, not a token
|
||||||
|
"sk-ant-short", # too short
|
||||||
|
"re_test_register_sends_verification", # windy-pro's fake Resend key
|
||||||
|
"ANTHROPIC_API_KEY=", # a name, not a value
|
||||||
|
])
|
||||||
|
def test_non_secrets_are_not_flagged(text):
|
||||||
|
assert sg.scan_line("x", text) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_anthropic_key_is_not_double_counted_as_openai():
|
||||||
|
assert [k for k, _ in sg.scan_line("x", "sk-ant-" + "q" * 40)] == ["anthropic key"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_is_by_hash_only():
|
||||||
|
F = sg.cg.Finding
|
||||||
|
fake = F("tests/t.py", 3, "telegram bot token", f"telegram bot token #{ss.h8(TG)}")
|
||||||
|
real = F("tests/t.py", 9, "telegram bot token", "telegram bot token #deadbeef")
|
||||||
|
allow = {"windy-chat": {"hashes": {ss.h8(TG)}, "paths": []}}
|
||||||
|
assert sg._drop_allowed("windy-chat", [fake, real], allow) == [real]
|
||||||
|
assert sg._drop_allowed("windy-mail", [fake], allow) == [fake]
|
||||||
|
|
||||||
|
|
||||||
|
def test_private_key_blocks_are_allowed_by_path_never_by_hash():
|
||||||
|
F = sg.cg.Finding
|
||||||
|
hdr = "private key block #" + ss.h8("-----BEGIN PRIVATE KEY-----")
|
||||||
|
test_key = F("tests/keys/test.pem", 1, "private key block", hdr)
|
||||||
|
prod_key = F("deploy/prod.pem", 1, "private key block", hdr)
|
||||||
|
allow = {"r": {"hashes": {hdr.rsplit("#", 1)[1]}, "paths": [("tests/keys/*", {"private key block"})]}}
|
||||||
|
assert sg._drop_allowed("r", [test_key, prod_key], allow) == [prod_key]
|
||||||
|
|
||||||
|
|
||||||
|
def test_path_allow_cannot_cover_real_token_kinds(tmp_path):
|
||||||
|
bad = tmp_path / "a.yml"
|
||||||
|
bad.write_text("allow:\n - repo: r\n paths: [tests/*]\n kinds: [telegram bot token]\n reason: no\n")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
sg.load_allow(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_shipped_allow_file_never_excuses_the_real_leaked_tokens():
|
||||||
|
a = sg.load_allow()
|
||||||
|
every = set().union(*(v["hashes"] for v in a.values())) if a else set()
|
||||||
|
assert not {"1354fc9b", "d49dc2ba"} & every # real (now revoked) credentials: remove, never allow
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_file_loads_and_needs_reasons(tmp_path):
|
||||||
|
assert isinstance(sg.load_allow(), dict)
|
||||||
|
bad = tmp_path / "a.yml"
|
||||||
|
bad.write_text("allow:\n - repo: r\n hashes: [abcd1234]\n")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
sg.load_allow(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def test_block_and_warn(monkeypatch):
|
||||||
|
f = sg.cg.Finding("a.py", 1, "github token", "github token #abcd1234")
|
||||||
|
monkeypatch.setattr(sg, "MODE", "block")
|
||||||
|
assert sg.status_for([f], False)[0] == "failure"
|
||||||
|
assert sg.status_for([], False, grant=[f])[0] == "success"
|
||||||
|
monkeypatch.setattr(sg, "MODE", "warn")
|
||||||
|
state, desc, _ = sg.status_for([f], True)
|
||||||
|
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 secret-shaped string in tree")
|
||||||
|
|
||||||
|
|
||||||
|
def test_public_scan_excuses_by_hash_and_by_path():
|
||||||
|
spec = importlib.util.spec_from_file_location("public_secret_scan", ROOT / "scripts" / "public_secret_scan.py")
|
||||||
|
ps = importlib.util.module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(ps)
|
||||||
|
allow = {"windy-agent": {"hashes": {"aaaa1111"}, "paths": [("tests/keys/*", {"private key block"})]}}
|
||||||
|
assert ps.excused("windy-agent", "openai key", "aaaa1111", ["tests/x.py"], allow)
|
||||||
|
assert not ps.excused("windy-agent", "telegram bot token", "1354fc9b", ["tests/test_log_redaction.py"], allow)
|
||||||
|
assert ps.excused("windy-agent", "private key block", "ffff0000", ["tests/keys/a.pem"], allow)
|
||||||
|
# a PEM header anywhere outside the allowed paths still counts
|
||||||
|
assert not ps.excused("windy-agent", "private key block", "ffff0000", ["tests/keys/a.pem", "deploy/k.pem"], allow)
|
||||||
|
assert not ps.excused("other", "openai key", "aaaa1111", ["x"], allow)
|
||||||
|
|
||||||
|
|
||||||
|
HEX32 = "0123456789abcdef" * 2 # synthetic
|
||||||
|
|
||||||
|
|
||||||
|
def test_twilio_shapes_hash_only_and_no_md5_noise():
|
||||||
|
kinds = lambda t: [k for k, _ in ss.find(t)] # noqa: E731
|
||||||
|
assert kinds(f'TWILIO_AUTH_TOKEN = "{HEX32}"') == ["32-hex secret assignment"]
|
||||||
|
assert kinds(f"auth_token: {HEX32}") == ["32-hex secret assignment"]
|
||||||
|
assert kinds("AC" + HEX32) == ["twilio sid/api key"]
|
||||||
|
assert kinds("SK" + HEX32) == ["twilio sid/api key"]
|
||||||
|
# plain md5 / uuid-without-dashes / a 64-hex sha256 are NOT secrets by shape
|
||||||
|
assert kinds(f"md5 = {HEX32}") == []
|
||||||
|
assert kinds(f"checksum_key = {HEX32}{HEX32}") == []
|
||||||
|
assert kinds(f"name = 'x{HEX32}'") == []
|
||||||
|
# the hash is of the value alone, so renaming the variable keeps the same allow hash
|
||||||
|
a = ss.find(f"A_TOKEN={HEX32}")[0][1]
|
||||||
|
b = ss.find(f"OTHER_SECRET: '{HEX32}'")[0][1]
|
||||||
|
assert a == b == ss.h8(HEX32)
|
||||||
|
assert HEX32 not in repr(ss.find(f"A_TOKEN={HEX32}"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_warn_kinds_do_not_block(monkeypatch):
|
||||||
|
f = sg.cg.Finding("a.py", 1, "32-hex secret assignment", "32-hex secret assignment #abcd1234")
|
||||||
|
monkeypatch.setattr(sg, "MODE", "block")
|
||||||
|
monkeypatch.setattr(sg, "WARN_KINDS", {"32-hex secret assignment"})
|
||||||
|
assert sg.status_for([f], True)[0] == "success"
|
||||||
|
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
||||||
|
assert sg.status_for([f], True)[0] == "failure"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pypi_token_shape_hash_only():
|
||||||
|
tok = "pypi-AgE" + "Ab1_-" * 20 # synthetic
|
||||||
|
got = ss.find(f"password = {tok}")
|
||||||
|
assert [k for k, _ in got] == ["pypi token"] and got[0][1] == ss.h8(tok)
|
||||||
|
assert tok not in repr(got)
|
||||||
|
assert ss.find("pypi-AgE-too-short") == []
|
||||||
106
api/tests/test_secret_scan.py
Normal file
106
api/tests/test_secret_scan.py
Normal file
@@ -0,0 +1,106 @@
|
|||||||
|
"""secret-scan + env-names: findings carry label/location/hash, NEVER a value or fragment."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPTS = ROOT / "scripts"
|
||||||
|
# Synthetic, random-looking, never real. FAKE_LB is in the fake lockbox; TWI matches a shape.
|
||||||
|
FAKE_LB = "k7Xv2QpLm9RtZw4HnB8dYc3S"
|
||||||
|
TWI = "9f3a7c1e5b2d48806a1f4e7d2c9b0835"
|
||||||
|
|
||||||
|
|
||||||
|
def run(script, *args, env=None):
|
||||||
|
e = {**os.environ, **(env or {})}
|
||||||
|
r = subprocess.run([sys.executable, str(SCRIPTS / script), *args], capture_output=True, text=True, env=e)
|
||||||
|
return r.returncode, r.stdout + r.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def no_fragment(out: str, value: str, n: int = 6):
|
||||||
|
assert value not in out
|
||||||
|
for i in range(len(value) - n + 1):
|
||||||
|
assert value[i:i + n] not in out, f"fragment of the value leaked at {i}"
|
||||||
|
|
||||||
|
|
||||||
|
def seeded(tmp_path):
|
||||||
|
lb = tmp_path / "lockbox.md"
|
||||||
|
lb.write_text(f"FAKE_VENDOR_API_KEY={FAKE_LB}\nNOTE: nothing here\n")
|
||||||
|
repo = tmp_path / "repo"
|
||||||
|
repo.mkdir()
|
||||||
|
g = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True) # noqa: E731
|
||||||
|
g("init", "-q", "-b", "main")
|
||||||
|
g("config", "user.email", "t@t")
|
||||||
|
g("config", "user.name", "t")
|
||||||
|
(repo / "app.py").write_text(f'KEY = "{FAKE_LB}"\nTWILIO_AUTH_TOKEN = "{TWI}"\n')
|
||||||
|
g("add", "-A")
|
||||||
|
g("commit", "-qm", "add secrets")
|
||||||
|
(repo / "app.py").write_text("KEY = None\n") # removed from HEAD, still in history
|
||||||
|
g("commit", "-qam", "remove")
|
||||||
|
return lb, repo
|
||||||
|
|
||||||
|
|
||||||
|
def test_tree_scan_labels_locations_no_value(tmp_path):
|
||||||
|
lb, repo = seeded(tmp_path)
|
||||||
|
(repo / "live.py").write_text(f'x = "{FAKE_LB}"\n')
|
||||||
|
rc, out = run("secret_scan.py", str(repo / "live.py"), env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb)})
|
||||||
|
assert rc == 1
|
||||||
|
assert "live.py:1" in out and "lockbox:FAKE_VENDOR_API_KEY" in out
|
||||||
|
no_fragment(out, FAKE_LB)
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_finds_removed_secret_with_commit(tmp_path):
|
||||||
|
lb, repo = seeded(tmp_path)
|
||||||
|
rc, out = run("secret_scan.py", str(repo), "--history", env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb)})
|
||||||
|
assert rc == 1
|
||||||
|
assert "app.py:1" in out and "commit=" in out and "lockbox:FAKE_VENDOR_API_KEY" in out
|
||||||
|
assert "app.py:2" in out and "32-hex secret assignment" in out
|
||||||
|
no_fragment(out, FAKE_LB)
|
||||||
|
no_fragment(out, TWI)
|
||||||
|
|
||||||
|
|
||||||
|
def test_repo_flag_clones_scans_and_cleans_up(tmp_path):
|
||||||
|
lb, repo = seeded(tmp_path)
|
||||||
|
cache = tmp_path / "home"
|
||||||
|
(cache / ".cache").mkdir(parents=True)
|
||||||
|
rc, out = run("secret_scan.py", "--repo", str(repo),
|
||||||
|
env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb), "HOME": str(cache)})
|
||||||
|
assert rc == 1 and "app.py:1" in out
|
||||||
|
no_fragment(out, FAKE_LB)
|
||||||
|
assert not [p for p in (cache / ".cache").iterdir() if p.name.startswith("secret-scan-")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_tree_and_bad_input(tmp_path):
|
||||||
|
(tmp_path / "ok.txt").write_text("hello world\n")
|
||||||
|
rc, out = run("secret_scan.py", str(tmp_path / "ok.txt"), "--no-lockbox")
|
||||||
|
assert rc == 0 and "0 finding(s)" in out
|
||||||
|
rc, out = run("secret_scan.py", str(tmp_path), "--history", "--no-lockbox")
|
||||||
|
assert rc == 2 and "needs a git repo" in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_env_names_never_prints_values(tmp_path):
|
||||||
|
a = tmp_path / "a.env"
|
||||||
|
b = tmp_path / "b.env"
|
||||||
|
a.write_text(f"# c\nexport DB_PASSWORD={FAKE_LB}\nTOKEN=\"{TWI}\"\nEMPTY=\nONLY_A=1\n")
|
||||||
|
b.write_text(f"DB_PASSWORD={FAKE_LB}\nTOKEN=different-value-here\nONLY_B=2\n")
|
||||||
|
rc, out = run("env_names.py", str(a), "--hash")
|
||||||
|
assert rc == 0 and "DB_PASSWORD" in out and "set" in out and "empty" in out and "len=24" in out
|
||||||
|
assert "sha256:" in out
|
||||||
|
no_fragment(out, FAKE_LB)
|
||||||
|
no_fragment(out, TWI, 7)
|
||||||
|
rc, out = run("env_names.py", str(a), "--compare", str(b))
|
||||||
|
assert "DB_PASSWORD" in out and "SAME" in out and "DIFFERENT" in out
|
||||||
|
assert "only-in-A" in out and "only-in-B" in out
|
||||||
|
no_fragment(out, FAKE_LB)
|
||||||
|
rc, out = run("env_names.py", str(tmp_path / "missing.env"))
|
||||||
|
assert rc == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_shapes_are_the_guards_shapes():
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
import secret_scan as sc
|
||||||
|
import secret_shapes as ss
|
||||||
|
assert sc.ss is ss # one source of shapes: a new guard shape is automatically a scan shape
|
||||||
@@ -45,3 +45,8 @@ allow:
|
|||||||
- repo: windy-git
|
- repo: windy-git
|
||||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||||
reason: "The guard's own pattern list and this file."
|
reason: "The guard's own pattern list and this file."
|
||||||
|
|
||||||
|
- repo: windy-mind
|
||||||
|
paths: ["*"]
|
||||||
|
matches: [':11434']
|
||||||
|
reason: "Windy Mind IS the compute door (endpoint + key); it may call Ollama. Only the Ollama port is allowed here, any provider host/SDK in Mind still flags."
|
||||||
|
|||||||
59
ci/secret-guard-allow.yml
Normal file
59
ci/secret-guard-allow.yml
Normal file
@@ -0,0 +1,59 @@
|
|||||||
|
# Secret guard allow-list: KNOWN FAKE values that look like secrets (test
|
||||||
|
# fixtures, docs). Allowed BY HASH (sha256[:8] of the value), so a real secret
|
||||||
|
# in the same file still flags. Private-key blocks (the match is only the BEGIN
|
||||||
|
# line, same hash everywhere) are allowed by PATH + kind instead.
|
||||||
|
# Every entry MUST say why. Owner: Windy Git lane (13); changes via the orchestrator.
|
||||||
|
# Triage 09-24 (values never printed): each hash checked against every version of
|
||||||
|
# the lockbox; fakes judged by impossible length for the kind (real Anthropic keys
|
||||||
|
# ~108 chars, OpenAI 51 or 160+), fake-words, or identity with upstream public
|
||||||
|
# fixtures. NOT allowed, remove instead: 1354fc9b (old @Windy_0_bot token) and
|
||||||
|
# d49dc2ba (old Anthropic key), both real and revoked, in public windy-agent.
|
||||||
|
allow:
|
||||||
|
- repo: windy-code
|
||||||
|
hashes: [ac9265e5, 46eb1235]
|
||||||
|
reason: "Upstream microsoft/vscode terminalEnvironment.test.ts fixtures (identical hash upstream; public)."
|
||||||
|
- repo: windy-code
|
||||||
|
paths: ["build/azure-pipelines/common/publish.ts"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "Upstream VS Code build script (PEM header string in code, not a key)."
|
||||||
|
- repo: windy-agent
|
||||||
|
hashes: [a9235a6d, dd6a2baa, 02c362d8, a6f6ff79, f7503b21, d0c94833, 4ab092e3, e3aa1eb8, 833382ee]
|
||||||
|
reason: "Redaction/sanitizer test fixtures; lengths impossible for real Anthropic/OpenAI keys; never in the lockbox."
|
||||||
|
- repo: windy-agent
|
||||||
|
hashes: [89bd408f, b8c94b72, bf23cdf5, d1d85dfe, e3e07f06]
|
||||||
|
reason: "09-24 #395 replacement fixtures (each contains FAKE; token-SHAPED on purpose so redaction tests prove real tokens are scrubbed); verified by Windy Agent sha-for-sha against every lockbox version: never real. Weekly scan 09-28."
|
||||||
|
- repo: windy-agent
|
||||||
|
paths: ["tests/test_agent_keys.py"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "Test-generated key material for agent-key tests."
|
||||||
|
- repo: windy-mind
|
||||||
|
hashes: [f8a630b2]
|
||||||
|
reason: "Provider test fixture (27 chars; a real Anthropic key is ~108)."
|
||||||
|
- repo: windy-pro
|
||||||
|
hashes: [756de8d8, 7828319d, 1a5d44a2]
|
||||||
|
reason: ".env.production.example placeholder + crash-summary test fixtures (AWS doc EXAMPLE key shape, short fake Slack token)."
|
||||||
|
- repo: windy-pro
|
||||||
|
paths: ["account-server/docs/oauth-providers.md"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "Docs show the PEM header format; no key material."
|
||||||
|
- repo: windytalk
|
||||||
|
hashes: [baf8656a]
|
||||||
|
reason: "Diagnostics redaction test fixture (fake-word in value)."
|
||||||
|
- repo: eternitas
|
||||||
|
paths: ["tests/golden_vectors/**", "tests/test_soul_vault_key_separation.py"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "Test vectors and throwaway keys for signature/vault tests."
|
||||||
|
- repo: windy-drops
|
||||||
|
paths: ["tools/conformance/test-keys/*"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "Conformance-suite test keys (named test-private.pem)."
|
||||||
|
- repo: windy-git
|
||||||
|
paths: ["api/tests/test_secret_guard.py"]
|
||||||
|
kinds: [private key block]
|
||||||
|
reason: "The guard's own test uses a PEM header string as a sample."
|
||||||
|
- repo: windy-code
|
||||||
|
hashes: ["23f32607"]
|
||||||
|
reason: "VS Code OSS extensions' package.json aiKey: Microsoft's public telemetry (App Insights) key, shipped in every VS Code build; not a Windy credential."
|
||||||
|
- repo: windytalk
|
||||||
|
hashes: ["c4189d79"]
|
||||||
|
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
||||||
14
deploy/systemd/windygit-state-backup.service
Normal file
14
deploy/systemd/windygit-state-backup.service
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2)
|
||||||
|
After=network-online.target docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/srv/windygit/src
|
||||||
|
# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass
|
||||||
|
# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD).
|
||||||
|
EnvironmentFile=/srv/windygit/src/.env
|
||||||
|
ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=idle
|
||||||
|
TimeoutStartSec=3h
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[Service]
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||||
11
deploy/systemd/windygit-state-backup.timer
Normal file
11
deploy/systemd/windygit-state-backup.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Nightly Windy Git state backup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# 03:07 local, clear of the git-bundle backup at 04:17.
|
||||||
|
OnCalendar=*-*-* 03:07:00
|
||||||
|
Persistent=true
|
||||||
|
RandomizedDelaySec=300
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[Service]
|
||||||
|
# Orchestrator 09-24: secret-guard BLOCKS lane-owned findings; Grant-owned stay WARN.
|
||||||
|
Environment=SECRET_GUARD_MODE=block
|
||||||
@@ -32,7 +32,7 @@ services:
|
|||||||
|
|
||||||
gitea:
|
gitea:
|
||||||
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
||||||
image: docker.io/gitea/gitea:1.24.6
|
image: docker.io/gitea/gitea:1.24.7
|
||||||
environment:
|
environment:
|
||||||
GITEA__database__DB_TYPE: postgres
|
GITEA__database__DB_TYPE: postgres
|
||||||
GITEA__database__HOST: db:5432
|
GITEA__database__HOST: db:5432
|
||||||
|
|||||||
33
docs/RESTORE-DRILL.md
Normal file
33
docs/RESTORE-DRILL.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Restoring Windy Git from the encrypted state backup
|
||||||
|
|
||||||
|
What is backed up (`scripts/backup_state.sh`, restic repo `s3:…/windy-git-backups/restic`, tag `windygit-state`):
|
||||||
|
both Postgres databases (`gitea`, `windygit`, custom-format dumps + globals), the whole Gitea data root
|
||||||
|
(`/srv/windygit/git`: config, jwt, attachments, avatars, templates AND the bare repositories),
|
||||||
|
`/srv/windygit/src/.env`, `deploy/runner/.env`, `/etc/cloudflared`, the windygit systemd drop-ins.
|
||||||
|
NOT in it: the restic password itself (lockbox `RESTIC_WINDYGIT_PASSWORD`) and the R2 access key
|
||||||
|
(scoped token `windy-git-r2-scoped`, lockbox). Never print either: use `lockbox-get KEY FILE`.
|
||||||
|
|
||||||
|
## Drill (any machine with restic + docker; proven on Veron 2026-10-01, counts identical)
|
||||||
|
export RESTIC_PASSWORD_FILE=<0600 file from lockbox-get RESTIC_WINDYGIT_PASSWORD>
|
||||||
|
export AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… # from lockbox-get, into env, not echoed
|
||||||
|
export RESTIC_REPOSITORY=s3:https://<R2 account>.r2.cloudflarestorage.com/windy-git-backups/restic
|
||||||
|
restic snapshots --tag windygit-state
|
||||||
|
restic restore latest --tag windygit-state --target /var/tmp/wg-drill
|
||||||
|
docker run -d --name wg-drill-pg -e POSTGRES_PASSWORD=<random> -e POSTGRES_USER=drill postgres:16-alpine
|
||||||
|
for db in gitea windygit; do
|
||||||
|
docker exec wg-drill-pg psql -U drill -d postgres -c "create database $db"
|
||||||
|
docker exec -i wg-drill-pg pg_restore -U drill -d $db --no-owner --no-privileges \
|
||||||
|
< /var/tmp/wg-drill/var/backups/windygit-state/$db.dump
|
||||||
|
done
|
||||||
|
# compare row counts with live (or with the last known): repository, issue, pull_request, "user",
|
||||||
|
# external_login_user, access_token, action_run, action_run_job
|
||||||
|
docker rm -f wg-drill-pg; rm -rf /var/tmp/wg-drill
|
||||||
|
|
||||||
|
## Real disaster (Veron lost)
|
||||||
|
1. New Linux host with Docker, a Cloudflare tunnel connector, the repo (`git clone` from GitHub: windy-git).
|
||||||
|
2. `restic restore latest --tag windygit-state --target /` (puts /srv/windygit/git, the .env files, /etc/cloudflared back).
|
||||||
|
3. `docker compose -p windy-git up -d db`, then pg_restore both dumps into it (as above, into the real db names/owner from `.env`).
|
||||||
|
4. `docker compose -p windy-git up -d` + `deploy/runner` runners; re-register runners if the token changed.
|
||||||
|
5. Verify: `/api/healthz`, Windy SSO login, `git ls-remote`, one CI run. GitHub is still the source of truth for code,
|
||||||
|
so repo content can also be re-synced from there; the database is what only this backup holds.
|
||||||
|
Retention: 14 daily / 8 weekly / 6 monthly (prune on Sundays). Integrity: every run does `restic check --read-data-subset=2%`.
|
||||||
66
scripts/backup_state.sh
Executable file
66
scripts/backup_state.sh
Executable file
@@ -0,0 +1,66 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Nightly STATE backup: everything git bundles do NOT hold (SOTU 10-01: 625 issues/PRs, users,
|
||||||
|
# SSO links, CI history, settings lived on one unbacked-up host). Encrypted restic repo in R2.
|
||||||
|
# - Postgres: every database (custom-format dump, restore-listable) + globals
|
||||||
|
# - Gitea config/data (app.ini, jwt, attachments, avatars, templates) + the bare repositories
|
||||||
|
# - the deploy .env files, systemd drop-ins and the cloudflared tunnel config (needed to rebuild)
|
||||||
|
# The restic password lives in /etc/windygit/restic.pass (root 600) AND the lockbox
|
||||||
|
# (RESTIC_WINDYGIT_PASSWORD): a lost Veron must not lose the backups. NEVER echo env/values here.
|
||||||
|
# Restore: docs/RESTORE-DRILL.md. Bounded: every docker exec runs under `timeout` (a hung
|
||||||
|
# runc exec in the IO stall wedged the sync on 09-23).
|
||||||
|
set -euo pipefail
|
||||||
|
log() { echo "[backup_state $(date -u +%FT%TZ)] $*"; }
|
||||||
|
: "${R2_ACCOUNT_ID:?}" "${R2_ACCESS_KEY_ID:?}" "${R2_SECRET_ACCESS_KEY:?}"
|
||||||
|
PASSFILE="${RESTIC_PASSWORD_FILE:-/etc/windygit/restic.pass}"
|
||||||
|
[[ -s "$PASSFILE" ]] || { log "FATAL: $PASSFILE missing/empty: refusing to report a backup that did not happen"; exit 1; }
|
||||||
|
export RESTIC_PASSWORD_FILE="$PASSFILE"
|
||||||
|
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||||
|
export RESTIC_REPOSITORY="${RESTIC_REPOSITORY:-s3:https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com/${R2_BUCKET_BACKUPS:-windy-git-backups}/restic}"
|
||||||
|
DB="${WG_DB_CONTAINER:-windy-git-db-1}"
|
||||||
|
STAGE="${WG_STAGE:-/var/backups/windygit-state}"
|
||||||
|
GIT_ROOT="${GIT_DATA_ROOT:-/srv/windygit/git}"
|
||||||
|
umask 077
|
||||||
|
mkdir -p "$STAGE"; chmod 700 "$STAGE"; rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||||
|
|
||||||
|
# systemd gives units no $HOME, and restic wants a cache dir: pin one.
|
||||||
|
export RESTIC_CACHE_DIR="${RESTIC_CACHE_DIR:-/var/cache/windygit-restic}"; mkdir -p "$RESTIC_CACHE_DIR"
|
||||||
|
if ! err=$(restic cat config 2>&1 >/dev/null); then
|
||||||
|
# only a MISSING repo may be initialised; any other error (auth, network, wrong password) must stop here
|
||||||
|
if grep -qiE "does not exist|is there a repository|unable to open config file" <<<"$err"; then
|
||||||
|
log "initialising restic repo"; restic init >/dev/null
|
||||||
|
else
|
||||||
|
log "FATAL: restic cannot open the repository: $(head -c 300 <<<"$err" | tr '\n' ' ')"; exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
PGU=$(timeout 30 docker exec "$DB" printenv POSTGRES_USER)
|
||||||
|
[[ -n "$PGU" ]] || { log "FATAL: no POSTGRES_USER in $DB"; exit 1; }
|
||||||
|
dbs=$(timeout 60 docker exec "$DB" psql -U "$PGU" -Atc "select datname from pg_database where not datistemplate and datname<>'postgres' order by 1")
|
||||||
|
n=0
|
||||||
|
for d in $dbs; do
|
||||||
|
timeout 600 docker exec "$DB" pg_dump -U "$PGU" -Fc "$d" > "$STAGE/$d.dump"
|
||||||
|
# a dump that cannot be listed is not a backup
|
||||||
|
timeout 120 docker exec -i "$DB" pg_restore -l < "$STAGE/$d.dump" >/dev/null
|
||||||
|
[[ $(stat -c%s "$STAGE/$d.dump") -gt 1000 ]] || { log "FATAL: $d dump suspiciously small"; exit 1; }
|
||||||
|
n=$((n+1)); log "dumped $d ($(stat -c%s "$STAGE/$d.dump") bytes)"
|
||||||
|
done
|
||||||
|
[[ $n -ge 1 ]] || { log "FATAL: no databases dumped"; exit 1; }
|
||||||
|
timeout 120 docker exec "$DB" pg_dumpall -U "$PGU" --globals-only > "$STAGE/globals.sql"
|
||||||
|
|
||||||
|
paths=("$STAGE" "$GIT_ROOT" /srv/windygit/src/.env /srv/windygit/src/deploy/runner/.env /etc/cloudflared)
|
||||||
|
for p in /etc/systemd/system/windygit-*.service.d /etc/windygit; do [[ -e $p ]] && paths+=("$p"); done
|
||||||
|
# restic.pass itself is excluded: the password never rides in its own backup
|
||||||
|
snap=$(restic backup --tag windygit-state --host windygit-veron --quiet --json \
|
||||||
|
--exclude "$GIT_ROOT/gitea/log" --exclude "$GIT_ROOT/gitea/queues" --exclude "$GIT_ROOT/gitea/tmp" \
|
||||||
|
--exclude "$GIT_ROOT/gitea/indexers" --exclude "$GIT_ROOT/gitea/actions_log" --exclude /etc/windygit/restic.pass \
|
||||||
|
"${paths[@]}" | python3 -c 'import sys,json
|
||||||
|
for l in sys.stdin:
|
||||||
|
d=json.loads(l)
|
||||||
|
if d.get("message_type")=="summary": print(d["snapshot_id"][:8])')
|
||||||
|
[[ -n "$snap" ]] || { log "FATAL: restic produced no snapshot"; exit 1; }
|
||||||
|
rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||||
|
restic check --read-data-subset=2% --quiet >/dev/null || { log "FATAL: restic check failed"; exit 1; }
|
||||||
|
if [[ $(date +%u) == 7 ]]; then
|
||||||
|
restic forget --tag windygit-state --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --prune --quiet >/dev/null
|
||||||
|
fi
|
||||||
|
echo "ok — state backed up ($n dbs, snapshot $snap)"
|
||||||
@@ -63,6 +63,9 @@ JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/gen
|
|||||||
|
|
||||||
RULES: list[tuple[str, re.Pattern]] = [
|
RULES: list[tuple[str, re.Pattern]] = [
|
||||||
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||||
|
# Grant via Boss 10-01: compute = Windy Mind. A NEW reference to an Ollama port (Veron's :11434) is a
|
||||||
|
# direct call around Mind's metering/caps. WARN-only, never red, and only for lines a PR ADDS.
|
||||||
|
("veron ollama", re.compile(r"(?::|%3[aA])11434(?![0-9])")),
|
||||||
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||||
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||||
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||||
@@ -70,6 +73,9 @@ RULES: list[tuple[str, re.Pattern]] = [
|
|||||||
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||||
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||||
]
|
]
|
||||||
|
# Kinds that never block (even in MODE=block) and are only judged on ADDED lines, never the baseline tree.
|
||||||
|
WARN_ONLY_KINDS = {"veron ollama"}
|
||||||
|
OLLAMA_MSG = "compute = Windy Mind (endpoint + key); do not call Veron's Ollama directly"
|
||||||
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||||
|
|
||||||
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||||
@@ -253,7 +259,8 @@ def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> li
|
|||||||
allow = load_allow()
|
allow = load_allow()
|
||||||
fp = _fingerprint(allow)
|
fp = _fingerprint(allow)
|
||||||
if is_default_head:
|
if is_default_head:
|
||||||
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
return cached_scan(f"tree:{repo}:{sha}:{fp}",
|
||||||
|
lambda: [f for f in scan_tree(repo, bare, sha, allow) if f.kind not in WARN_ONLY_KINDS])
|
||||||
return cached_scan(
|
return cached_scan(
|
||||||
f"pr:{repo}:{sha}:{fp}",
|
f"pr:{repo}:{sha}:{fp}",
|
||||||
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||||
@@ -268,6 +275,11 @@ def status_for(findings: list[Finding], whole_tree: bool,
|
|||||||
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
||||||
09-23: his desktop work is never blocked by us)."""
|
09-23: his desktop work is never blocked by us)."""
|
||||||
scope = "in tree" if whole_tree else "added"
|
scope = "in tree" if whole_tree else "added"
|
||||||
|
soft = [f for f in findings if f.kind in WARN_ONLY_KINDS]
|
||||||
|
findings = [f for f in findings if f.kind not in WARN_ONLY_KINDS]
|
||||||
|
if not findings and not grant and soft:
|
||||||
|
f = soft[0]
|
||||||
|
return "success", f"⚠ WARN: new Veron Ollama ref {f.path}:{f.line}. {OLLAMA_MSG}"[:140], f
|
||||||
if not findings and grant:
|
if not findings and grant:
|
||||||
g, n = grant[0], len(grant)
|
g, n = grant[0], len(grant)
|
||||||
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
||||||
|
|||||||
23
scripts/drill_cross_host.sh
Executable file
23
scripts/drill_cross_host.sh
Executable file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed.
|
||||||
|
# Usage: bash drill_cross_host.sh (needs lockbox keys RESTIC_WINDYGIT_PASSWORD, WINDYGIT_R2_ACCESS_KEY_ID, WINDYGIT_R2_SECRET_ACCESS_KEY, WINDYGIT_R2_ENDPOINT)
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX)
|
||||||
|
trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT
|
||||||
|
lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null
|
||||||
|
lockbox-get WINDYGIT_R2_ACCESS_KEY_ID "$W/ak" >/dev/null; lockbox-get WINDYGIT_R2_SECRET_ACCESS_KEY "$W/sk" >/dev/null; lockbox-get WINDYGIT_R2_ENDPOINT "$W/ep" >/dev/null
|
||||||
|
export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")"
|
||||||
|
export RESTIC_REPOSITORY="s3:$(cat "$W/ep")/windy-git-backups/restic"
|
||||||
|
restic snapshots --tag windygit-state --compact | tail -3
|
||||||
|
restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet
|
||||||
|
ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}'
|
||||||
|
docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null
|
||||||
|
for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done
|
||||||
|
for db in gitea windygit; do
|
||||||
|
docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db"
|
||||||
|
docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true
|
||||||
|
done
|
||||||
|
for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do
|
||||||
|
echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")"
|
||||||
|
done
|
||||||
|
echo "cross-host drill OK (cleaned up)"
|
||||||
153
scripts/env_names.py
Normal file
153
scripts/env_names.py
Normal file
@@ -0,0 +1,153 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""env-names: list environment variable NAMES only (Boss ruling 10-01, house rule 10).
|
||||||
|
|
||||||
|
env-names <docker container | systemd unit | env file> [--host H] [--sudo] [--hash]
|
||||||
|
env-names A --compare B [--host H] [--host2 H2]
|
||||||
|
|
||||||
|
Prints NAME, set|empty, and value LENGTH. Never a value or fragment. --hash adds sha256[:8]
|
||||||
|
(compare two places for equality; a hash of a weak value can be guessed, so use it for
|
||||||
|
real secrets only). --compare prints SAME / DIFFERENT / only-in-A / only-in-B per name
|
||||||
|
(equality by full-value hash, nothing else shown). Values live in memory only.
|
||||||
|
Targets: an existing file (dotenv style) | a docker container name | a systemd unit
|
||||||
|
(Environment= + EnvironmentFile=; --sudo to read root-only files). --host runs the docker /
|
||||||
|
systemctl / file read over ssh (alias from ~/.ssh/config).
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shlex
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
KV = ("=",)
|
||||||
|
|
||||||
|
|
||||||
|
def run(cmd: list[str], host: str | None, sudo: bool = False) -> tuple[int, str]:
|
||||||
|
if sudo:
|
||||||
|
cmd = ["sudo", "-n", *cmd]
|
||||||
|
if host:
|
||||||
|
cmd = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", host, shlex.join(cmd)]
|
||||||
|
r = subprocess.run(cmd, capture_output=True, text=True, errors="ignore", timeout=60)
|
||||||
|
return r.returncode, r.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def parse_dotenv(text: str) -> dict[str, str]:
|
||||||
|
out: dict[str, str] = {}
|
||||||
|
for raw in text.splitlines():
|
||||||
|
line = raw.strip()
|
||||||
|
if not line or line.startswith("#") or "=" not in line:
|
||||||
|
continue
|
||||||
|
if line.startswith("export "):
|
||||||
|
line = line[7:].lstrip()
|
||||||
|
k, v = line.split("=", 1)
|
||||||
|
k = k.strip()
|
||||||
|
v = v.strip()
|
||||||
|
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
|
||||||
|
v = v[1:-1]
|
||||||
|
if k.replace("_", "").isalnum() and not k[0].isdigit():
|
||||||
|
out[k] = v
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def from_file(path: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||||
|
if host or sudo:
|
||||||
|
rc, out = run(["cat", path], host, sudo)
|
||||||
|
return parse_dotenv(out) if rc == 0 else None
|
||||||
|
try:
|
||||||
|
with open(path, errors="ignore") as fh:
|
||||||
|
return parse_dotenv(fh.read())
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def from_docker(name: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||||
|
rc, out = run(["docker", "inspect", "-f", "{{json .Config.Env}}", name], host, sudo)
|
||||||
|
if rc != 0 or not out.strip():
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
items = json.loads(out)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
return {k: v for k, _, v in (i.partition("=") for i in (items or []))}
|
||||||
|
|
||||||
|
|
||||||
|
def from_systemd(unit: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||||
|
rc, out = run(["systemctl", "show", unit, "-p", "Environment", "-p", "EnvironmentFiles"], host)
|
||||||
|
if rc != 0 or "LoadState=not-found" in out:
|
||||||
|
return None
|
||||||
|
env: dict[str, str] = {}
|
||||||
|
files: list[str] = []
|
||||||
|
for line in out.splitlines():
|
||||||
|
if line.startswith("Environment="):
|
||||||
|
for tok in shlex.split(line[len("Environment="):]):
|
||||||
|
k, _, v = tok.partition("=")
|
||||||
|
env[k] = v
|
||||||
|
elif line.startswith("EnvironmentFiles="):
|
||||||
|
f = line[len("EnvironmentFiles="):].split(" (")[0].strip().lstrip("-")
|
||||||
|
if f:
|
||||||
|
files.append(f)
|
||||||
|
for f in files: # later files override earlier, like systemd
|
||||||
|
d = from_file(f, host, sudo)
|
||||||
|
if d is None:
|
||||||
|
print(f"# note: EnvironmentFile {f} unreadable (try --sudo)", file=sys.stderr)
|
||||||
|
else:
|
||||||
|
env.update(d)
|
||||||
|
return env
|
||||||
|
|
||||||
|
|
||||||
|
def load(target: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||||
|
if (not host and os.path.isfile(target)) or target.startswith(("/", "./", "~")):
|
||||||
|
return from_file(os.path.expanduser(target), host, sudo)
|
||||||
|
if target.endswith((".service", ".timer", ".socket")):
|
||||||
|
return from_systemd(target, host, sudo)
|
||||||
|
return from_docker(target, host, sudo) or from_systemd(target, host, sudo)
|
||||||
|
|
||||||
|
|
||||||
|
def sh(v: str) -> str:
|
||||||
|
return hashlib.sha256(v.encode()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(prog="env-names", description="env var NAMES only")
|
||||||
|
ap.add_argument("target")
|
||||||
|
ap.add_argument("--host")
|
||||||
|
ap.add_argument("--host2", help="ssh host for the --compare target")
|
||||||
|
ap.add_argument("--sudo", action="store_true")
|
||||||
|
ap.add_argument("--hash", action="store_true", help="add sha256[:8] per variable")
|
||||||
|
ap.add_argument("--compare", metavar="TARGET2")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
env = load(a.target, a.host, a.sudo)
|
||||||
|
if env is None:
|
||||||
|
print(f"error: could not read {a.target!r} (file, docker container or systemd unit)")
|
||||||
|
return 2
|
||||||
|
if a.compare:
|
||||||
|
env2 = load(a.compare, a.host2 or a.host, a.sudo)
|
||||||
|
if env2 is None:
|
||||||
|
print(f"error: could not read {a.compare!r}")
|
||||||
|
return 2
|
||||||
|
for k in sorted(set(env) | set(env2)):
|
||||||
|
if k not in env2:
|
||||||
|
print(f"{k:<40} only-in-A")
|
||||||
|
elif k not in env:
|
||||||
|
print(f"{k:<40} only-in-B")
|
||||||
|
else:
|
||||||
|
print(f"{k:<40} {'SAME' if sh(env[k]) == sh(env2[k]) else 'DIFFERENT'}"
|
||||||
|
f" (len {len(env[k])} vs {len(env2[k])})")
|
||||||
|
return 0
|
||||||
|
for k in sorted(env):
|
||||||
|
v = env[k]
|
||||||
|
extra = f" sha256:{sh(v)[:8]}" if a.hash and v else ""
|
||||||
|
print(f"{k:<40} {'set ' if v else 'empty'} len={len(v)}{extra}")
|
||||||
|
print(f"# {len(env)} variable(s); values never printed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except Exception as e: # never a traceback
|
||||||
|
print(f"error: {type(e).__name__}")
|
||||||
|
sys.exit(2)
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Live status of the repo guards (compute-guard + ci-hygiene) as one markdown page.
|
"""Live status of the repo guards (compute-guard + ci-hygiene + secret-guard) as one markdown page.
|
||||||
|
|
||||||
Scans every bridged repo's DEFAULT branch with both guards and renders what is
|
Scans every bridged repo's DEFAULT branch with both guards and renders what is
|
||||||
left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml:
|
left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml:
|
||||||
@@ -24,6 +24,7 @@ import yaml
|
|||||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
import ci_hygiene as hy # noqa: E402
|
import ci_hygiene as hy # noqa: E402
|
||||||
import compute_guard as cg # noqa: E402
|
import compute_guard as cg # noqa: E402
|
||||||
|
import secret_guard as sgd # noqa: E402
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml"))
|
OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml"))
|
||||||
@@ -32,7 +33,7 @@ REPOS = os.environ.get("BRIDGE_REPOS", "").split() or [
|
|||||||
"windy-connect", "windy-drops", "windy-code-web", "windy-code", "windy-traveler",
|
"windy-connect", "windy-drops", "windy-code-web", "windy-code", "windy-traveler",
|
||||||
"windy-registry", "eternitas", "windy-translate", "windytranslate-site", "windytraveler-site",
|
"windy-registry", "eternitas", "windy-translate", "windytranslate-site", "windytraveler-site",
|
||||||
"windy-hand", "windy-cloud-sites", "windy-cloud-domains", "windy-cloud-vps", "windytalk",
|
"windy-hand", "windy-cloud-sites", "windy-cloud-domains", "windy-cloud-vps", "windytalk",
|
||||||
"windy-pro", "windy-mind", "windy-git"]
|
"windy-pro", "windy-mind", "windy-git", "windy-inbox"]
|
||||||
# Owner lane per repo = the session name to message (orchestrator routing, 09-23 ~22:45Z:
|
# Owner lane per repo = the session name to message (orchestrator routing, 09-23 ~22:45Z:
|
||||||
# hub/account-server/dashboard/site -> "Windy Hub"; windy-calendar only -> "Windy Calender";
|
# hub/account-server/dashboard/site -> "Windy Hub"; windy-calendar only -> "Windy Calender";
|
||||||
# windy-admin/telemetry -> "Windy Admin"). windy-pro here = its server/web side; the desktop
|
# windy-admin/telemetry -> "Windy Admin"). windy-pro here = its server/web side; the desktop
|
||||||
@@ -45,7 +46,8 @@ OWNERS = {
|
|||||||
"windy-code-web": "Windy Code", "windy-code": "Windy Code", "windy-traveler": "Windy Traveler",
|
"windy-code-web": "Windy Code", "windy-code": "Windy Code", "windy-traveler": "Windy Traveler",
|
||||||
"windytraveler-site": "Windy Traveler", "eternitas": "Eternitas", "windy-translate": "Windy Translate",
|
"windytraveler-site": "Windy Traveler", "eternitas": "Eternitas", "windy-translate": "Windy Translate",
|
||||||
"windytranslate-site": "Windy Translate", "windy-hand": "Windy Hand", "windytalk": "Windy Talk",
|
"windytranslate-site": "Windy Translate", "windy-hand": "Windy Hand", "windytalk": "Windy Talk",
|
||||||
"windy-pro": "Windy Hub", "windy-mind": "WIndy Mind", "windy-git": "Windy Git"}
|
"windy-pro": "Windy Hub", "windy-mind": "WIndy Mind", "windy-git": "Windy Git",
|
||||||
|
"windy-inbox": "Windy Drops"} # Windy Inbox build lead (09-24)
|
||||||
JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
|
JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
|
||||||
|
|
||||||
|
|
||||||
@@ -100,10 +102,11 @@ def scan(repo: str, owned: list[dict]):
|
|||||||
return None
|
return None
|
||||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||||
sha = cg._git(bare, "rev-parse", head).strip()
|
sha = cg._git(bare, "rev-parse", head).strip()
|
||||||
out = {"sha": sha, "compute": [], "hygiene": []}
|
out = {"sha": sha, "compute": [], "hygiene": [], "secrets": []}
|
||||||
texts: dict[str, str] = {}
|
texts: dict[str, str] = {}
|
||||||
for key, fs in (("compute", cg.check(repo, sha, head, True) or []),
|
for key, fs in (("compute", cg.check(repo, sha, head, True) or []),
|
||||||
("hygiene", hy.check(repo, sha, head, True) or [])):
|
("hygiene", hy.check(repo, sha, head, True) or []),
|
||||||
|
("secrets", sgd.check(repo, sha, head, True) or [])):
|
||||||
for f in fs:
|
for f in fs:
|
||||||
job = None
|
job = None
|
||||||
if "/workflows/" in f.path:
|
if "/workflows/" in f.path:
|
||||||
@@ -116,27 +119,29 @@ def scan(repo: str, owned: list[dict]):
|
|||||||
|
|
||||||
def render(results: dict) -> str:
|
def render(results: dict) -> str:
|
||||||
now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime())
|
now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime())
|
||||||
lane = {k: 0 for k in ("compute", "hygiene")}
|
lane = {k: 0 for k in ("compute", "hygiene", "secrets")}
|
||||||
grant = {k: 0 for k in ("compute", "hygiene")}
|
grant = {k: 0 for k in ("compute", "hygiene", "secrets")}
|
||||||
for r in results.values():
|
for r in results.values():
|
||||||
for k in lane:
|
for k in lane:
|
||||||
lane[k] += sum(1 for _, _, g in r[k] if not g)
|
lane[k] += sum(1 for _, _, g in r.get(k, []) if not g)
|
||||||
grant[k] += sum(1 for _, _, g in r[k] if g)
|
grant[k] += sum(1 for _, _, g in r.get(k, []) if g)
|
||||||
L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)",
|
L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)",
|
||||||
"_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. "
|
"_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. "
|
||||||
"Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "",
|
"Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "",
|
||||||
"| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|",
|
"| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|",
|
||||||
f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |",
|
f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |",
|
||||||
f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |",
|
f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |",
|
||||||
"", "## By repo (lane-owned)", "| Repo | owner | head | compute | hygiene | first items |", "|---|---|---|---|---|---|"]
|
f"| secret-guard (no credentials in repos; hash only) | {lane['secrets']} | {grant['secrets']} | {'✅ YES' if lane['secrets'] == 0 else '❌ not yet'} |",
|
||||||
|
"", "## By repo (lane-owned)", "| Repo | owner | head | compute | hygiene | secrets | first items |", "|---|---|---|---|---|---|---|"]
|
||||||
for repo, r in sorted(results.items()):
|
for repo, r in sorted(results.items()):
|
||||||
c = [x for x in r["compute"] if not x[2]]
|
c = [x for x in r["compute"] if not x[2]]
|
||||||
h = [x for x in r["hygiene"] if not x[2]]
|
h = [x for x in r["hygiene"] if not x[2]]
|
||||||
items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (c + h)[:3]) or "clean ✅"
|
s = [x for x in r.get("secrets", []) if not x[2]]
|
||||||
L.append(f"| {repo} | {OWNERS.get(repo, '?')} | {r['sha'][:7]} | {len(c)} | {len(h)} | {items} |")
|
items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (s + c + h)[:3]) or "clean ✅"
|
||||||
|
L.append(f"| {repo} | {OWNERS.get(repo, '?')} | {r['sha'][:7]} | {len(c)} | {len(h)} | {len(s)} | {items} |")
|
||||||
L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"]
|
L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"]
|
||||||
g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene")
|
g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene", "secrets")
|
||||||
for f, job, own in r[k] if own]
|
for f, job, own in r.get(k, []) if own]
|
||||||
L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"]
|
L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"]
|
||||||
return "\n".join(L) + "\n"
|
return "\n".join(L) + "\n"
|
||||||
|
|
||||||
|
|||||||
14
scripts/install_secret_tools.sh
Executable file
14
scripts/install_secret_tools.sh
Executable file
@@ -0,0 +1,14 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Install the shared hash-only secret tools on THIS machine (Windy 0): secret-scan + env-names.
|
||||||
|
# Source of truth is this repo (scripts/); re-run after a pull to update.
|
||||||
|
set -euo pipefail
|
||||||
|
here=$(cd "$(dirname "$0")" && pwd)
|
||||||
|
dest="$HOME/.local/share/secret-tools"
|
||||||
|
mkdir -p "$dest" "$HOME/.local/bin"
|
||||||
|
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$dest/"
|
||||||
|
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py"; do
|
||||||
|
n=${pair%%:*}; f=${pair##*:}
|
||||||
|
printf '#!/usr/bin/env bash\nexec python3 "%s/%s" "$@"\n' "$dest" "$f" > "$HOME/.local/bin/$n"
|
||||||
|
chmod 755 "$HOME/.local/bin/$n"
|
||||||
|
done
|
||||||
|
echo "installed secret-scan, env-names and lockbox-put (shapes from secret_shapes.py, same as secret-guard)"
|
||||||
141
scripts/lockbox_put.py
Normal file
141
scripts/lockbox_put.py
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""lockbox-put: add ONE secret to the lockbox by PR, without anyone reading, printing or
|
||||||
|
grepping the lockbox (Boss rule 10-01; Windy Hub ruling).
|
||||||
|
|
||||||
|
lockbox-put KEY FILE [--lane NAME] [--note TEXT]
|
||||||
|
|
||||||
|
KEY exact name, ^[A-Z][A-Z0-9_]{2,63}$ . FILE a 0600 file you own (not a symlink) whose
|
||||||
|
content is the value (one line). Appends ONE line `- **`KEY`**: `<value>`` (the format
|
||||||
|
lockbox-get reads) under a new heading at the END of ACCESS_LOCKBOX.md in a fresh temp clone,
|
||||||
|
on a new branch, and opens a kit-army-config PR. Append-only: the diff is verified to be one
|
||||||
|
file, additions only, before pushing. REFUSES if KEY already exists (a bool computed in
|
||||||
|
memory; no line, value or location is ever printed). Reviewers see the KEY NAME + lane only
|
||||||
|
if they look at the diff; the PR body never carries the value. Never echoes the value.
|
||||||
|
Env (tests): LOCKBOX_PUT_REPO=<clone url/path>, LOCKBOX_PUT_NO_PR=1.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import datetime as dt
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import stat
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
REPO = os.environ.get("LOCKBOX_PUT_REPO", "https://github.com/sneakyfree/kit-army-config.git")
|
||||||
|
SLUG = "sneakyfree/kit-army-config"
|
||||||
|
KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]{2,63}$")
|
||||||
|
VAL_RE = re.compile(r"^[A-Za-z0-9._~+/=:@%,-]{8,512}$") # no backtick, quote, space or newline
|
||||||
|
|
||||||
|
|
||||||
|
def die(msg: str, code: int = 2):
|
||||||
|
print(f"lockbox-put: {msg}")
|
||||||
|
sys.exit(code)
|
||||||
|
|
||||||
|
|
||||||
|
def git(cwd: str, *a: str, quiet=True) -> subprocess.CompletedProcess:
|
||||||
|
# stderr is dropped: git/gh errors can echo URLs; stdout only when we need it.
|
||||||
|
return subprocess.run(["git", "-C", cwd, *a], capture_output=True, text=True, errors="ignore")
|
||||||
|
|
||||||
|
|
||||||
|
def key_exists(clone: str, key: str) -> bool:
|
||||||
|
"""True if KEY is already defined anywhere lockbox-get reads. Bool only, nothing printed."""
|
||||||
|
pat_env = re.compile(r"^" + re.escape(key) + r"=")
|
||||||
|
pat_md = re.compile(r"^\s*[-*]?\s*\*\*`" + re.escape(key) + r"`\*\*\s*:")
|
||||||
|
paths = [Path(clone, "ACCESS_LOCKBOX.md")] + [
|
||||||
|
Path(dp, f) for dp, _d, fs in os.walk(Path(clone, "secrets")) for f in fs if f.endswith(".env")]
|
||||||
|
for p in paths:
|
||||||
|
try:
|
||||||
|
with open(p, errors="ignore") as fh:
|
||||||
|
for line in fh:
|
||||||
|
if pat_env.match(line) or pat_md.match(line):
|
||||||
|
return True
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(prog="lockbox-put")
|
||||||
|
ap.add_argument("key")
|
||||||
|
ap.add_argument("file")
|
||||||
|
ap.add_argument("--lane", default=os.environ.get("LOCKBOX_LANE", "a lane"))
|
||||||
|
ap.add_argument("--note", default="")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
if not KEY_RE.match(a.key):
|
||||||
|
die("KEY must match ^[A-Z][A-Z0-9_]{2,63}$")
|
||||||
|
try:
|
||||||
|
st = os.lstat(a.file)
|
||||||
|
except OSError:
|
||||||
|
die("FILE not found")
|
||||||
|
if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode):
|
||||||
|
die("FILE must be a regular file (not a symlink)")
|
||||||
|
if st.st_uid != os.getuid() or (st.st_mode & 0o077):
|
||||||
|
die("FILE must be owned by you and mode 0600")
|
||||||
|
with open(a.file) as fh:
|
||||||
|
value = fh.read().strip()
|
||||||
|
if not VAL_RE.match(value):
|
||||||
|
die("value must be one line of 8-512 chars from [A-Za-z0-9._~+/=:@%,-] (no spaces, quotes, backticks)")
|
||||||
|
note = re.sub(r"[`\n\r]", " ", a.note)[:160]
|
||||||
|
lane = re.sub(r"[^A-Za-z0-9 ._-]", "", a.lane)[:40]
|
||||||
|
tmp = tempfile.mkdtemp(prefix="lockbox-put-", dir=str(Path.home() / ".cache") if (Path.home() / ".cache").is_dir() else None)
|
||||||
|
os.chmod(tmp, 0o700)
|
||||||
|
clone = os.path.join(tmp, "k")
|
||||||
|
try:
|
||||||
|
if subprocess.run(["git", "clone", "-q", "--depth", "1", REPO, clone],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0:
|
||||||
|
die("clone failed (details withheld: URLs can carry tokens)")
|
||||||
|
if key_exists(clone, a.key):
|
||||||
|
die(f"{a.key} already exists: refusing to overwrite (append-only; pick a new KEY)", 3)
|
||||||
|
lb = Path(clone, "ACCESS_LOCKBOX.md")
|
||||||
|
if not lb.is_file():
|
||||||
|
die("ACCESS_LOCKBOX.md not found in the repo")
|
||||||
|
today = dt.date.today().isoformat()
|
||||||
|
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%d%H%M")
|
||||||
|
branch = f"lockbox-put/{a.key.lower()}-{stamp}"
|
||||||
|
with open(lb, "a") as fh:
|
||||||
|
fh.write(f"\n## 🗝️ {a.key} (added {today} by {lane} via lockbox-put)\n")
|
||||||
|
fh.write(f"- **`{a.key}`**: `{value}`\n")
|
||||||
|
if note:
|
||||||
|
fh.write(f"- **Note:** {note}\n")
|
||||||
|
git(clone, "checkout", "-q", "-b", branch)
|
||||||
|
git(clone, "add", "ACCESS_LOCKBOX.md")
|
||||||
|
ns = git(clone, "diff", "--cached", "--numstat").stdout.split()
|
||||||
|
# numstat: <added> <deleted> <path>; exactly one file, no deletions
|
||||||
|
if len(ns) != 3 or ns[1] != "0" or ns[2] != "ACCESS_LOCKBOX.md":
|
||||||
|
die("diff is not a pure append to ACCESS_LOCKBOX.md: aborting, nothing pushed")
|
||||||
|
msg = f"lockbox: add {a.key} (via lockbox-put, {lane})\n\nCo-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>"
|
||||||
|
if git(clone, "-c", "user.name=lockbox-put", "-c", "user.email=lockbox-put@windy.invalid",
|
||||||
|
"commit", "-q", "-m", msg).returncode != 0:
|
||||||
|
die("commit failed")
|
||||||
|
if git(clone, "push", "-q", "origin", branch).returncode != 0:
|
||||||
|
die("push failed (details withheld)")
|
||||||
|
if os.environ.get("LOCKBOX_PUT_NO_PR"):
|
||||||
|
print(f"ok: pushed branch {branch} ({a.key}); PR skipped")
|
||||||
|
return 0
|
||||||
|
body = (f"Adds exactly one key: `{a.key}` (by {lane}). Append-only, one file, no deletions "
|
||||||
|
f"(verified before push). Review by KEY NAME only; do not paste the value anywhere.\n\n"
|
||||||
|
f"{note}\n\n🤖 Generated with [Claude Code](https://claude.com/claude-code)")
|
||||||
|
r = subprocess.run(["gh", "pr", "create", "-R", SLUG, "--head", branch, "--base", "main",
|
||||||
|
"--title", f"lockbox: add {a.key} ({lane})", "--body", body],
|
||||||
|
capture_output=True, text=True)
|
||||||
|
if r.returncode != 0:
|
||||||
|
die("branch pushed but `gh pr create` failed; open the PR for the branch by hand")
|
||||||
|
print(f"ok: {a.key} added via PR {r.stdout.strip().splitlines()[-1]}")
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except SystemExit:
|
||||||
|
raise
|
||||||
|
except Exception as e: # never a traceback: it could carry data
|
||||||
|
print(f"lockbox-put: error: {type(e).__name__}")
|
||||||
|
sys.exit(2)
|
||||||
@@ -53,7 +53,8 @@ REPOS = os.environ.get(
|
|||||||
"windy-chat windy-mail windy-calendar Windy-Clone WindyCloud windy-search windy-connect"
|
"windy-chat windy-mail windy-calendar Windy-Clone WindyCloud windy-search windy-connect"
|
||||||
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
||||||
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
||||||
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind",
|
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
||||||
|
" windy-inbox windy-text windy-call windy-cell",
|
||||||
).split()
|
).split()
|
||||||
|
|
||||||
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
||||||
@@ -74,7 +75,9 @@ MIRROR_TAG = "[GH#"
|
|||||||
# Posting them would put a permanent red X on every commit, and a signal that is
|
# Posting them would put a permanent red X on every commit, and a signal that is
|
||||||
# always red trains everyone to ignore red. Not posted until a rootless builder
|
# always red trains everyone to ignore red. Not posted until a rootless builder
|
||||||
# exists; that is a decision, recorded in docs/CUTOVER.md, not a failure.
|
# exists; that is a decision, recorded in docs/CUTOVER.md, not a failure.
|
||||||
NO_DAEMON_JOB = re.compile(r"docker", re.IGNORECASE)
|
# ...but NOT the no-Docker smoke jobs that replaced them (option A, 09-23):
|
||||||
|
# windy-search's "Boot smoke (no Docker)" matched plain `docker` and was hidden.
|
||||||
|
NO_DAEMON_JOB = re.compile(r"(?<!no )(?<!no-)(?<!without )docker", re.IGNORECASE)
|
||||||
# Image-build jobs whose NAME doesn't say docker (orchestrator 09-23, option A:
|
# Image-build jobs whose NAME doesn't say docker (orchestrator 09-23, option A:
|
||||||
# each lane converts the job to a no-Docker smoke test; until then it is not
|
# each lane converts the job to a no-Docker smoke test; until then it is not
|
||||||
# posted). Format: "repo:workflow/job,...;repo2:...".
|
# posted). Format: "repo:workflow/job,...;repo2:...".
|
||||||
@@ -369,6 +372,7 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
|
|
||||||
GUARD_CTX = "windy-git/compute-guard"
|
GUARD_CTX = "windy-git/compute-guard"
|
||||||
HYGIENE_CTX = "windy-git/ci-hygiene"
|
HYGIENE_CTX = "windy-git/ci-hygiene"
|
||||||
|
SECRET_CTX = "windy-git/secret-guard"
|
||||||
|
|
||||||
|
|
||||||
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
@@ -376,6 +380,11 @@ def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head
|
|||||||
_post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head)
|
_post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head)
|
||||||
|
|
||||||
|
|
||||||
|
def post_secret_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
|
"""No live credential in a bridged repo (leak hunt 09-24). Findings carry sha256[:8] only."""
|
||||||
|
_post_guard("secret_guard", SECRET_CTX, repo, sha, default_branch, is_default_head)
|
||||||
|
|
||||||
|
|
||||||
def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
"""House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only)."""
|
"""House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only)."""
|
||||||
_post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head)
|
_post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head)
|
||||||
@@ -435,6 +444,7 @@ def main() -> int:
|
|||||||
post_statuses(repo, sha)
|
post_statuses(repo, sha)
|
||||||
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
||||||
post_ci_hygiene(repo, sha, default_branch, sha == default_head)
|
post_ci_hygiene(repo, sha, default_branch, sha == default_head)
|
||||||
|
post_secret_guard(repo, sha, default_branch, sha == default_head)
|
||||||
except Exception as e: # one repo's failure must not hide the others'
|
except Exception as e: # one repo's failure must not hide the others'
|
||||||
print(f" FAILED {repo}: {e}")
|
print(f" FAILED {repo}: {e}")
|
||||||
failed = 1
|
failed = 1
|
||||||
|
|||||||
118
scripts/public_secret_scan.py
Normal file
118
scripts/public_secret_scan.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Weekly: every PUBLIC repo in Grant's GitHub accounts, full history (every object,
|
||||||
|
reachable or not, incl. PR refs), for secret-shaped strings. Leak hunt 09-24: a
|
||||||
|
real bot token sat in a public test fixture for five months.
|
||||||
|
|
||||||
|
Output is hash + location only, never a value (house rule 10). Known fakes are
|
||||||
|
excused by ci/secret-guard-allow.yml (same file as secret-guard; the repo NAME is
|
||||||
|
matched across accounts). Runs on Veron as user1-gpu (gh is logged in there):
|
||||||
|
|
||||||
|
python3 scripts/public_secret_scan.py [--out FILE] [--owners a,b,...]
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import fnmatch
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import secret_shapes as ss # noqa: E402
|
||||||
|
|
||||||
|
OWNERS = ["sneakyfree", "VERONTECH", "Windstorm-Institute", "Windstorm-Labs", "Public-Streamer"]
|
||||||
|
WORK = Path.home() / "leakscan" / "public"
|
||||||
|
MAX_BLOB = 20_000_000
|
||||||
|
|
||||||
|
|
||||||
|
def _run(*a: str) -> subprocess.CompletedProcess:
|
||||||
|
return subprocess.run(a, capture_output=True)
|
||||||
|
|
||||||
|
|
||||||
|
def blob_hits(bare: Path) -> dict[str, list[tuple[str, str]]]:
|
||||||
|
"""{blob: [(kind, hash8)]} over every blob object in the repo."""
|
||||||
|
chk = _run("git", "-C", str(bare), "cat-file", "--batch-all-objects",
|
||||||
|
"--batch-check=%(objectname) %(objecttype) %(objectsize)")
|
||||||
|
blobs = [p[0] for p in (ln.split() for ln in chk.stdout.decode().splitlines())
|
||||||
|
if len(p) == 3 and p[1] == "blob" and int(p[2]) < MAX_BLOB]
|
||||||
|
if not blobs:
|
||||||
|
return {}
|
||||||
|
import threading
|
||||||
|
p = subprocess.Popen(["git", "-C", str(bare), "cat-file", "--batch"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
def feed() -> None: # separate thread: writing everything first deadlocks (both pipes fill)
|
||||||
|
p.stdin.write(("\n".join(blobs) + "\n").encode())
|
||||||
|
p.stdin.close()
|
||||||
|
threading.Thread(target=feed, daemon=True).start()
|
||||||
|
out: dict[str, list[tuple[str, str]]] = {}
|
||||||
|
for _ in blobs:
|
||||||
|
hdr = p.stdout.readline().split()
|
||||||
|
data = p.stdout.read(int(hdr[2]))
|
||||||
|
p.stdout.read(1)
|
||||||
|
found = ss.find(data.decode("utf-8", "ignore"))
|
||||||
|
if found:
|
||||||
|
out[hdr[0].decode()] = found
|
||||||
|
p.wait()
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def locate(bare: Path, blob: str) -> dict:
|
||||||
|
lg = _run("git", "-C", str(bare), "log", "--all", "--format=@@%H %cI", "--name-only",
|
||||||
|
f"--find-object={blob}").stdout.decode()
|
||||||
|
commits, paths = [], set()
|
||||||
|
for line in lg.splitlines():
|
||||||
|
if line.startswith("@@"):
|
||||||
|
commits.append(line[2:].split())
|
||||||
|
elif line.strip():
|
||||||
|
paths.add(line.strip())
|
||||||
|
head = _run("git", "-C", str(bare), "ls-tree", "-r", "HEAD").stdout.decode()
|
||||||
|
first = commits[-1] if commits else ["unreachable", "-"]
|
||||||
|
return {"paths": sorted(paths), "first_commit": first[0][:10], "first_date": first[1],
|
||||||
|
"in_head": blob in head}
|
||||||
|
|
||||||
|
|
||||||
|
def excused(repo: str, kind: str, h: str, paths: list[str], allow: dict) -> bool:
|
||||||
|
a = allow.get(repo) or {"hashes": set(), "paths": []}
|
||||||
|
if kind in {"private key block"}:
|
||||||
|
return bool(paths) and all(any(kind in k and fnmatch.fnmatch(p, g) for g, k in a["paths"]) for p in paths)
|
||||||
|
return h in a["hashes"]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--out", default=str(WORK / "latest.json"))
|
||||||
|
ap.add_argument("--owners", default=",".join(OWNERS))
|
||||||
|
args = ap.parse_args()
|
||||||
|
import secret_guard as sg
|
||||||
|
allow = sg.load_allow()
|
||||||
|
WORK.mkdir(parents=True, exist_ok=True)
|
||||||
|
rows, scanned, errors = [], [], []
|
||||||
|
for owner in args.owners.split(","):
|
||||||
|
lst = _run("gh", "repo", "list", owner, "--limit", "1000", "--visibility", "public", "--json", "name")
|
||||||
|
for r in json.loads(lst.stdout or b"[]"):
|
||||||
|
name = r["name"]
|
||||||
|
bare = WORK / owner / f"{name}.git"
|
||||||
|
if bare.exists():
|
||||||
|
c = _run("git", "-C", str(bare), "remote", "update", "--prune")
|
||||||
|
else:
|
||||||
|
bare.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
c = _run("gh", "repo", "clone", f"{owner}/{name}", str(bare), "--", "--mirror", "-q")
|
||||||
|
if c.returncode:
|
||||||
|
errors.append(f"{owner}/{name}")
|
||||||
|
continue
|
||||||
|
scanned.append(f"{owner}/{name}")
|
||||||
|
for blob, found in blob_hits(bare).items():
|
||||||
|
loc = locate(bare, blob)
|
||||||
|
for kind, h in sorted(set(found)):
|
||||||
|
rows.append({"repo": f"{owner}/{name}", "kind": kind, "hash8": h, **loc,
|
||||||
|
"excused": excused(name, kind, h, loc["paths"], allow)})
|
||||||
|
Path(args.out).write_text(json.dumps({"scanned": scanned, "errors": errors, "rows": rows}, indent=1))
|
||||||
|
new = [r for r in rows if not r["excused"]]
|
||||||
|
print(f"scanned {len(scanned)} public repos, {len(errors)} errors, {len(rows)} secret-shaped, {len(new)} NOT excused")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -16,6 +16,8 @@
|
|||||||
# itself, so Windy Git is never left behind GitHub.
|
# itself, so Windy Git is never left behind GitHub.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
repo="${1:?repo}"; branch="${2:?branch}"; want="${3:?sha prefix}"
|
repo="${1:?repo}"; branch="${2:?branch}"; want="${3:?sha prefix}"
|
||||||
|
# wg-q lives in the INVOKING user's ~/bin; under sudo, ~ is /root.
|
||||||
|
WGQ="${WGQ:-$(getent passwd "${SUDO_USER:-$USER}" | cut -d: -f6)/bin/wg-q}"
|
||||||
# Gitea stores repositories LOWERCASED on disk (WindyCloud -> windycloud.git).
|
# Gitea stores repositories LOWERCASED on disk (WindyCloud -> windycloud.git).
|
||||||
G="sudo docker exec -u git windy-git-gitea-1 git -C /data/git/repositories/windyadmin/${repo,,}.git"
|
G="sudo docker exec -u git windy-git-gitea-1 git -C /data/git/repositories/windyadmin/${repo,,}.git"
|
||||||
|
|
||||||
@@ -43,7 +45,7 @@ until [ "$(systemctl show windygit-sync -p ExecMainStartTimestampMonotonic --val
|
|||||||
done
|
done
|
||||||
echo "restored by sync: ${branch} = ${head:0:7}"
|
echo "restored by sync: ${branch} = ${head:0:7}"
|
||||||
sleep 5
|
sleep 5
|
||||||
~/bin/wg-q <<SQL
|
"$WGQ" <<SQL
|
||||||
select ar.index, ar.workflow_id, ar.event, ar.status, to_char(to_timestamp(ar.created),'HH24:MI:SS')
|
select ar.index, ar.workflow_id, ar.event, ar.status, to_char(to_timestamp(ar.created),'HH24:MI:SS')
|
||||||
from action_run ar join repository r on r.id = ar.repo_id
|
from action_run ar join repository r on r.id = ar.repo_id
|
||||||
where r.name = '${repo}' and ar.commit_sha = '${head}' order by ar.id desc limit 6;
|
where r.name = '${repo}' and ar.commit_sha = '${head}' order by ar.id desc limit 6;
|
||||||
|
|||||||
131
scripts/secret_guard.py
Normal file
131
scripts/secret_guard.py
Normal file
@@ -0,0 +1,131 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Secret guard: no live credential lands in a bridged repo (leak hunt 09-24).
|
||||||
|
|
||||||
|
Same walker, cache and GitHub posting as compute_guard / ci_hygiene
|
||||||
|
(`windy-git/secret-guard`), but over EVERY text file, and a finding carries only
|
||||||
|
"<kind> #<sha256[:8]>", never the value (house rule 10). Known fakes are allowed
|
||||||
|
BY HASH in ci/secret-guard-allow.yml (repo + hashes + reason).
|
||||||
|
|
||||||
|
sudo python3 scripts/secret_guard.py report [repo ...]
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import compute_guard as cg # noqa: E402 (shared walker, cache)
|
||||||
|
import secret_shapes as ss # noqa: E402
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
ALLOW_FILE = Path(os.environ.get("SECRET_GUARD_ALLOW", ROOT / "ci" / "secret-guard-allow.yml"))
|
||||||
|
MODE = os.environ.get("SECRET_GUARD_MODE", "warn")
|
||||||
|
# Kinds that only WARN (rolled out warn-first); empty = every kind blocks in block mode.
|
||||||
|
WARN_KINDS = {k for k in os.environ.get("SECRET_GUARD_WARN_KINDS", "").split(",") if k}
|
||||||
|
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
||||||
|
|
||||||
|
|
||||||
|
def path_ok(path: str) -> bool:
|
||||||
|
return not NEVER.search(path)
|
||||||
|
|
||||||
|
|
||||||
|
# A private-key match is only its BEGIN line, so its hash is the same everywhere:
|
||||||
|
# those are allowed by PATH (entries with `paths` + `kinds`), everything else by HASH.
|
||||||
|
PATH_ONLY_KINDS = {"private key block"}
|
||||||
|
|
||||||
|
|
||||||
|
def load_allow(path: Path = ALLOW_FILE) -> dict[str, dict]:
|
||||||
|
"""{repo: {"hashes": {hash8}, "paths": [(glob, {kind})]}}; every entry needs a reason."""
|
||||||
|
data = yaml.safe_load(path.read_text()) if path.exists() else {}
|
||||||
|
out: dict[str, dict] = {}
|
||||||
|
for e in (data or {}).get("allow") or []:
|
||||||
|
if not (e.get("repo") and (e.get("hashes") or (e.get("paths") and e.get("kinds")))
|
||||||
|
and str(e.get("reason", "")).strip()):
|
||||||
|
raise ValueError(f"allow entry needs repo, hashes (or paths + kinds) and a reason: {e}")
|
||||||
|
if e.get("paths") and not set(e["kinds"]) <= PATH_ONLY_KINDS:
|
||||||
|
raise ValueError(f"path allows are only for {sorted(PATH_ONLY_KINDS)}: {e}")
|
||||||
|
r = out.setdefault(e["repo"], {"hashes": set(), "paths": []})
|
||||||
|
r["hashes"].update(str(h) for h in e.get("hashes") or [])
|
||||||
|
r["paths"] += [(g, set(e["kinds"])) for g in e.get("paths") or []]
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
||||||
|
return [(kind, f"{kind} #{h}") for kind, h in ss.find(text)]
|
||||||
|
|
||||||
|
|
||||||
|
def _drop_allowed(repo: str, findings, allow: dict[str, dict]):
|
||||||
|
a = allow.get(repo) or {"hashes": set(), "paths": []}
|
||||||
|
|
||||||
|
def ok(f) -> bool:
|
||||||
|
if f.kind in PATH_ONLY_KINDS:
|
||||||
|
return any(f.kind in kinds and fnmatch.fnmatch(f.path, g) for g, kinds in a["paths"])
|
||||||
|
return f.match.rsplit("#", 1)[-1] in a["hashes"]
|
||||||
|
return [f for f in findings if not ok(f)]
|
||||||
|
|
||||||
|
|
||||||
|
def check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
||||||
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
||||||
|
return None
|
||||||
|
allow = load_allow()
|
||||||
|
rules = hashlib.sha256(("|".join(rx.pattern for _, rx in ss.PATTERNS) + ss.PREFILTER).encode()).hexdigest()[:8]
|
||||||
|
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
||||||
|
if is_default_head:
|
||||||
|
fs = cg.cached_scan(f"sec-tree:{repo}:{sha}:{rules}",
|
||||||
|
lambda: cg.scan_tree(repo, bare, sha, [], prefilter=ss.PREFILTER, **kw))
|
||||||
|
else:
|
||||||
|
fs = cg.cached_scan(f"sec-pr:{repo}:{sha}:{rules}",
|
||||||
|
lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, [], **kw))
|
||||||
|
return _drop_allowed(repo, fs, allow)
|
||||||
|
|
||||||
|
|
||||||
|
def status_for(findings, whole_tree: bool, grant=()):
|
||||||
|
"""Same contract as the other guards. `grant` findings never block."""
|
||||||
|
scope = "in tree" if whole_tree else "added"
|
||||||
|
if not findings and grant:
|
||||||
|
g, n = grant[0], len(grant)
|
||||||
|
return "success", f"⚠ WARN (Grant-owned, not blocking): {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {g.path}:{g.line} {g.match}"[:140], g
|
||||||
|
if not findings:
|
||||||
|
return "success", f"OK: no secret-shaped strings {scope}", None
|
||||||
|
f, n = findings[0], len(findings)
|
||||||
|
soft = MODE != "block" or all(x.kind in WARN_KINDS for x in findings)
|
||||||
|
state = "success" if soft else "failure"
|
||||||
|
lead = "⚠ WARN (not blocking)" if soft else "BLOCKED"
|
||||||
|
if not soft:
|
||||||
|
f = next(x for x in findings if x.kind not in WARN_KINDS)
|
||||||
|
return state, f"{lead}: {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
||||||
|
|
||||||
|
|
||||||
|
def report(repos: list[str]) -> int:
|
||||||
|
allow = load_allow()
|
||||||
|
total = 0
|
||||||
|
for repo in repos:
|
||||||
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir():
|
||||||
|
continue
|
||||||
|
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||||
|
sha = cg._git(bare, "rev-parse", head).strip()
|
||||||
|
fs = _drop_allowed(repo, cg.scan_tree(repo, bare, sha, [], line_fn=scan_line, path_ok=path_ok,
|
||||||
|
prefilter=ss.PREFILTER), allow)
|
||||||
|
total += len(fs)
|
||||||
|
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} finding(s)")
|
||||||
|
for f in fs:
|
||||||
|
print(f" {f.path}:{f.line} {f.match}")
|
||||||
|
print(f"TOTAL {total}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
||||||
|
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
||||||
|
p.name.removesuffix(".git") for p in cg.WORK.glob("*.git"))
|
||||||
|
sys.exit(report(sys.argv[2:] or default))
|
||||||
|
sys.exit(__doc__)
|
||||||
210
scripts/secret_scan.py
Normal file
210
scripts/secret_scan.py
Normal file
@@ -0,0 +1,210 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""secret-scan: hash-only secret finder. Boss ruling 10-01 after three lanes printed secrets
|
||||||
|
into their own transcripts while hunting secrets (house rule 10).
|
||||||
|
|
||||||
|
secret-scan <path> [--history] [--repo <git url or path>] [--no-lockbox]
|
||||||
|
|
||||||
|
Reports `file:line` (and the commit with --history), WHICH lockbox entry matched (the KEY
|
||||||
|
NAME only) or which secret SHAPE matched (twilio, zai, aws, ...), plus a sha256[:8] of the
|
||||||
|
token for allow-listing. It NEVER prints, logs or writes a value or any fragment of one
|
||||||
|
(no context line, no masking). The lockbox is loaded in memory only. stdout only.
|
||||||
|
Exit 0 = clean, 1 = findings, 2 = usage/error.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import secret_shapes as ss # noqa: E402 (the SAME shapes as secret-guard)
|
||||||
|
|
||||||
|
HOME = Path.home()
|
||||||
|
LOCKBOX_PATHS = [p for p in os.environ.get("SECRET_SCAN_LOCKBOX_PATHS", "").split(":") if p] or [
|
||||||
|
str(HOME / "kit-army-config" / "secrets"), str(HOME / "kit-army-config" / "ACCESS_LOCKBOX.md")]
|
||||||
|
# Extra shapes that are scan-only (not in the blocking guard): label, regex.
|
||||||
|
EXTRA = [("zai key", re.compile(r"(?<![0-9a-f])[0-9a-f]{32}\.[A-Za-z0-9]{16}(?![A-Za-z0-9])"))]
|
||||||
|
SKIP_DIRS = {".git", "node_modules", "vendor", "third_party", "__pycache__", ".venv"}
|
||||||
|
MAX_BYTES = 5_000_000
|
||||||
|
|
||||||
|
RUN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_\-]{15,199}")
|
||||||
|
UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||||
|
NAME = re.compile(r"[\s>*`|-]*([A-Za-z][A-Za-z0-9_]{2,60})\s*[=:|]")
|
||||||
|
|
||||||
|
|
||||||
|
def h16(t: str) -> str:
|
||||||
|
return hashlib.sha256(t.encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def cands(line: str):
|
||||||
|
"""Token candidates: runs >=16 chars with a digit and a letter; git shas/uuids excluded."""
|
||||||
|
for chunk in re.split(r"[^A-Za-z0-9_\-.]+", line):
|
||||||
|
parts = [chunk, *chunk.split(".")] if "." in chunk else [chunk]
|
||||||
|
for p in parts:
|
||||||
|
for m in RUN.finditer(p):
|
||||||
|
t = m.group(0)
|
||||||
|
if not (re.search(r"\d", t) and re.search(r"[A-Za-z]", t)):
|
||||||
|
continue
|
||||||
|
if re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", t) or UUID.match(t.lower()):
|
||||||
|
continue
|
||||||
|
yield t
|
||||||
|
|
||||||
|
|
||||||
|
def load_lockbox() -> dict[str, set[str]]:
|
||||||
|
"""{hash16: {key-name labels}}; values never leave this dict."""
|
||||||
|
out: dict[str, set[str]] = {}
|
||||||
|
files: list[str] = []
|
||||||
|
for p in LOCKBOX_PATHS:
|
||||||
|
if os.path.isdir(p):
|
||||||
|
for root, _d, fs in os.walk(p):
|
||||||
|
files += [os.path.join(root, f) for f in fs]
|
||||||
|
elif os.path.isfile(p):
|
||||||
|
files.append(p)
|
||||||
|
for f in files:
|
||||||
|
try:
|
||||||
|
with open(f, errors="ignore") as fh:
|
||||||
|
for line in fh:
|
||||||
|
m = NAME.match(line)
|
||||||
|
label = m.group(1) if m else "?"
|
||||||
|
for t in cands(line):
|
||||||
|
out.setdefault(h16(t), set()).add(label)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def scan_line(line: str, lockbox: dict[str, set[str]]) -> list[tuple[str, str]]:
|
||||||
|
"""[(label, hash8)]: `shape:<kind>` and/or `lockbox:<NAMES>`. No value escapes."""
|
||||||
|
res: list[tuple[str, str]] = []
|
||||||
|
for kind, h in ss.find(line):
|
||||||
|
res.append((f"shape:{kind}", h))
|
||||||
|
for kind, rx in EXTRA:
|
||||||
|
for m in rx.finditer(line):
|
||||||
|
res.append((f"shape:{kind}", ss.h8(m.group(0))))
|
||||||
|
for t in cands(line):
|
||||||
|
k = h16(t)
|
||||||
|
if k in lockbox:
|
||||||
|
names = sorted(n for n in lockbox[k])
|
||||||
|
res.append(("lockbox:" + ",".join(names)[:70], k[:8]))
|
||||||
|
return sorted(set(res))
|
||||||
|
|
||||||
|
|
||||||
|
def is_text(path: Path) -> bool:
|
||||||
|
try:
|
||||||
|
with open(path, "rb") as fh:
|
||||||
|
return b"\0" not in fh.read(4096)
|
||||||
|
except OSError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def scan_tree(root: Path, lockbox):
|
||||||
|
files = [root] if root.is_file() else [
|
||||||
|
Path(dp) / f for dp, dn, fn in os.walk(root) for f in fn
|
||||||
|
if not set(Path(dp).relative_to(root).parts) & SKIP_DIRS]
|
||||||
|
for p in sorted(files):
|
||||||
|
try:
|
||||||
|
if p.stat().st_size > MAX_BYTES or not is_text(p):
|
||||||
|
continue
|
||||||
|
with open(p, errors="ignore") as fh:
|
||||||
|
for n, line in enumerate(fh, 1):
|
||||||
|
for label, h in scan_line(line, lockbox):
|
||||||
|
yield (str(p), n, None, label, h)
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
|
||||||
|
|
||||||
|
def git(repo: str, *a: str) -> subprocess.Popen:
|
||||||
|
return subprocess.Popen(["git", "--git-dir", repo, *a], stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL, text=True, errors="ignore")
|
||||||
|
|
||||||
|
|
||||||
|
def scan_history(gitdir: str, lockbox):
|
||||||
|
"""Every ADDED line on every ref (incl. PR refs). Oldest commit per (hash, file, line)."""
|
||||||
|
seen: dict[tuple, str] = {}
|
||||||
|
p = git(gitdir, "log", "--all", "-p", "-U0", "--no-color", "--format=@@C %h", "-a")
|
||||||
|
commit = path = None
|
||||||
|
ln = 0
|
||||||
|
for row in p.stdout: # type: ignore[union-attr]
|
||||||
|
if row.startswith("@@C "):
|
||||||
|
commit = row[4:].strip()
|
||||||
|
elif row.startswith("+++ "):
|
||||||
|
path = row[6:].strip() if row.startswith("+++ b/") else None
|
||||||
|
elif row.startswith("@@ "):
|
||||||
|
m = re.search(r"\+(\d+)", row)
|
||||||
|
ln = int(m.group(1)) - 1 if m else 0
|
||||||
|
elif row.startswith("+") and path:
|
||||||
|
ln += 1
|
||||||
|
for label, h in scan_line(row[1:], lockbox):
|
||||||
|
seen[(label, h, path, ln)] = commit or "?"
|
||||||
|
p.wait()
|
||||||
|
for (label, h, path, ln), c in sorted(seen.items(), key=lambda x: (x[0][2], x[0][3])):
|
||||||
|
yield (path, ln, c, label, h)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_gitdir(p: Path) -> str | None:
|
||||||
|
for cand in (p / ".git", p):
|
||||||
|
if (cand / "HEAD").exists() and ((cand / "objects").exists()):
|
||||||
|
return str(cand)
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(prog="secret-scan", description=__doc__.split("\n\n")[1] if __doc__ else "")
|
||||||
|
ap.add_argument("path", nargs="?", help="file, directory, or git repo (with --history)")
|
||||||
|
ap.add_argument("--history", action="store_true", help="scan every added line in all git history")
|
||||||
|
ap.add_argument("--repo", help="git URL or path to scan (mirror-cloned to a temp dir, then deleted)")
|
||||||
|
ap.add_argument("--no-lockbox", action="store_true", help="shapes only")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
if not (a.path or a.repo):
|
||||||
|
ap.print_usage()
|
||||||
|
return 2
|
||||||
|
lockbox = {} if a.no_lockbox else load_lockbox()
|
||||||
|
print(f"# secret-scan: {len(lockbox)} lockbox tokens in memory, values never printed", flush=True)
|
||||||
|
tmp = None
|
||||||
|
findings = 0
|
||||||
|
try:
|
||||||
|
if a.repo:
|
||||||
|
tmp = tempfile.mkdtemp(prefix="secret-scan-", dir=str(HOME / ".cache") if (HOME / ".cache").is_dir() else None)
|
||||||
|
os.chmod(tmp, 0o700)
|
||||||
|
target = os.path.join(tmp, "r.git")
|
||||||
|
rc = subprocess.run(["git", "clone", "-q", "--mirror", a.repo, target],
|
||||||
|
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
||||||
|
if rc != 0:
|
||||||
|
print("error: clone failed (details withheld: URLs can carry tokens)")
|
||||||
|
return 2
|
||||||
|
a.history = True
|
||||||
|
gitdir = target
|
||||||
|
elif a.history:
|
||||||
|
gitdir = resolve_gitdir(Path(a.path))
|
||||||
|
if not gitdir:
|
||||||
|
print("error: --history needs a git repo path")
|
||||||
|
return 2
|
||||||
|
if a.history:
|
||||||
|
for path, ln, c, label, h in scan_history(gitdir, lockbox):
|
||||||
|
findings += 1
|
||||||
|
print(f"{path}:{ln} commit={c} {label} #{h}")
|
||||||
|
else:
|
||||||
|
for path, ln, _c, label, h in scan_tree(Path(a.path), lockbox):
|
||||||
|
findings += 1
|
||||||
|
print(f"{path}:{ln} {label} #{h}")
|
||||||
|
finally:
|
||||||
|
if tmp:
|
||||||
|
shutil.rmtree(tmp, ignore_errors=True)
|
||||||
|
print(f"# {findings} finding(s)")
|
||||||
|
return 1 if findings else 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
sys.exit(130)
|
||||||
|
except Exception as e: # never a traceback: it could carry data
|
||||||
|
print(f"error: {type(e).__name__}")
|
||||||
|
sys.exit(2)
|
||||||
46
scripts/secret_shapes.py
Normal file
46
scripts/secret_shapes.py
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
"""Secret-shaped strings, shared by secret_guard (bridged repos) and
|
||||||
|
public_secret_scan (weekly, every public repo). A finding NEVER carries the value:
|
||||||
|
only its kind and sha256[:8] (house rule 10). Leak hunt 09-24: @Windy_0_bot's
|
||||||
|
token sat in a public repo's test fixture for five months."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import re
|
||||||
|
|
||||||
|
# (kind, regex). Order matters only for readability; each match is reported once.
|
||||||
|
PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||||
|
("telegram bot token", re.compile(r"(?<![0-9])[0-9]{8,10}:[A-Za-z0-9_-]{35}(?![A-Za-z0-9_-])")),
|
||||||
|
("github token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})\b")),
|
||||||
|
("aws access key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
|
||||||
|
("slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
||||||
|
("anthropic key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
|
||||||
|
("openai key", re.compile(r"\bsk-(?:proj-|svcacct-)?(?!ant-)[A-Za-z0-9_-]{32,}")),
|
||||||
|
("stripe live key", re.compile(r"\b[rs]k_live_[A-Za-z0-9]{20,}")),
|
||||||
|
("google api key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])")),
|
||||||
|
# Twilio (Windy Text 10-01: a live auth token sat in test files for months). An auth token
|
||||||
|
# is a bare 32-hex with no prefix, so it is only caught when ASSIGNED to a secret-ish name.
|
||||||
|
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
||||||
|
("32-hex secret assignment", re.compile(
|
||||||
|
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
||||||
|
("pypi token", re.compile(r"\bpypi-AgE[A-Za-z0-9_-]{50,}")),
|
||||||
|
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
||||||
|
]
|
||||||
|
|
||||||
|
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
||||||
|
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
||||||
|
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
||||||
|
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}|pypi-AgE")
|
||||||
|
|
||||||
|
|
||||||
|
def h8(value: str | bytes) -> str:
|
||||||
|
return hashlib.sha256(value.encode() if isinstance(value, str) else value).hexdigest()[:8]
|
||||||
|
|
||||||
|
|
||||||
|
def find(text: str) -> list[tuple[str, str]]:
|
||||||
|
"""[(kind, hash8)] for every secret-shaped string in `text`. Values never leave."""
|
||||||
|
out = []
|
||||||
|
for kind, rx in PATTERNS:
|
||||||
|
for m in rx.finditer(text):
|
||||||
|
out.append((kind, h8(m.group('v') if 'v' in rx.groupindex else m.group(0))))
|
||||||
|
return out
|
||||||
@@ -38,7 +38,7 @@ FAILED=0
|
|||||||
|
|
||||||
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
||||||
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
||||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro}"
|
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell}"
|
||||||
|
|
||||||
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
||||||
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
||||||
|
|||||||
Reference in New Issue
Block a user