Compare commits
13 Commits
8690c4f8d3
...
veron-olla
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd0400c371 | ||
|
|
4e7ab667ed | ||
|
|
f10db19316 | ||
|
|
3503894a1e | ||
|
|
fbab5c4669 | ||
|
|
1b552c0882 | ||
|
|
8ebc18c3bc | ||
|
|
9566826ce9 | ||
|
|
160a3d69ba | ||
|
|
7e28a23c22 | ||
|
|
1da4d39c0b | ||
|
|
53fbcfe78f | ||
|
|
1c552e94de |
@@ -229,3 +229,24 @@ def test_block_mode_never_blocks_grant_owned(monkeypatch):
|
||||
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
||||
lane = cg.Finding("a.py", 3, "provider host", "x")
|
||||
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
||||
|
||||
|
||||
def test_veron_ollama_warns_on_added_lines_and_never_blocks(monkeypatch):
|
||||
hits = cg.scan_line("app/llm.py", 'OLLAMA = "http://192.168.1.73:11434/api/generate"')
|
||||
assert [k for k, _ in hits] == ["veron ollama"]
|
||||
assert cg.scan_line("app/llm.py", 'port = 114345') == [] # not the port
|
||||
assert cg.scan_line("app/llm.py", "# was http://x:11434 (removed)") == [] # a comment is not a call
|
||||
f = cg.Finding("app/llm.py", 7, "veron ollama", ":11434")
|
||||
monkeypatch.setattr(cg, "MODE", "block")
|
||||
state, desc, _ = cg.status_for([f], whole_tree=False)
|
||||
assert state == "success" and desc.startswith("⚠ WARN: new Veron Ollama ref app/llm.py:7")
|
||||
assert "Windy Mind" in desc and len(desc) <= 140
|
||||
hard = cg.Finding("app/llm.py", 1, "provider host", "api.openai.com")
|
||||
assert cg.status_for([f, hard], whole_tree=False)[0] == "failure" # a real violation still blocks
|
||||
|
||||
|
||||
def test_ollama_in_added_pr_lines_only():
|
||||
diff = ("+++ b/svc/client.py\n@@ -0,0 +1,2 @@\n+import httpx\n"
|
||||
"+URL = 'http://veron:11434/api/chat'\n")
|
||||
got = cg.parse_added("some-repo", diff, [])
|
||||
assert [(f.kind, f.line) for f in got] == [("veron ollama", 2)]
|
||||
|
||||
77
api/tests/test_lockbox_put.py
Normal file
77
api/tests/test_lockbox_put.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""lockbox-put against a LOCAL fake lockbox repo only (never the real one)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
FAKE = "Zk3vQ8mT1pLw7Xn2Rb5Hd9Yc" # synthetic
|
||||
|
||||
|
||||
def sh(*a, cwd=None):
|
||||
return subprocess.run(a, cwd=cwd, check=True, capture_output=True, text=True)
|
||||
|
||||
|
||||
def make_remote(tmp_path):
|
||||
work = tmp_path / "seed"
|
||||
work.mkdir()
|
||||
sh("git", "init", "-q", "-b", "main", cwd=work)
|
||||
(work / "ACCESS_LOCKBOX.md").write_text("# LOCKBOX\n\n- **`EXISTING_KEY`**: `abcdefgh12345678`\nOLD_ENV_KEY=whatever123456\n")
|
||||
sh("git", "add", "-A", cwd=work)
|
||||
sh("git", "-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "seed", cwd=work)
|
||||
bare = tmp_path / "remote.git"
|
||||
sh("git", "clone", "-q", "--bare", str(work), str(bare))
|
||||
return bare
|
||||
|
||||
|
||||
def put(tmp_path, bare, key, content, mode=0o600):
|
||||
f = tmp_path / "val"
|
||||
f.write_text(content)
|
||||
os.chmod(f, mode)
|
||||
e = {**os.environ, "LOCKBOX_PUT_REPO": str(bare), "LOCKBOX_PUT_NO_PR": "1", "HOME": str(tmp_path / "h")}
|
||||
(tmp_path / "h" / ".cache").mkdir(parents=True, exist_ok=True)
|
||||
r = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_put.py"), key, str(f), "--lane", "test", "--note", "n"],
|
||||
capture_output=True, text=True, env=e)
|
||||
return r.returncode, r.stdout + r.stderr
|
||||
|
||||
|
||||
def test_appends_one_key_by_branch_and_never_echoes_value(tmp_path):
|
||||
bare = make_remote(tmp_path)
|
||||
rc, out = put(tmp_path, bare, "NEW_TEST_KEY", FAKE)
|
||||
assert rc == 0 and "pushed branch lockbox-put/new_test_key-" in out
|
||||
assert FAKE not in out and FAKE[:6] not in out
|
||||
br = [b.strip() for b in sh("git", "branch", "--list", "lockbox-put/*", cwd=bare).stdout.splitlines()]
|
||||
assert len(br) == 1
|
||||
diff = sh("git", "diff", "--numstat", f"main..{br[0]}", cwd=bare).stdout.split()
|
||||
assert diff[1] == "0" and diff[2] == "ACCESS_LOCKBOX.md" # additions only
|
||||
content = sh("git", "show", f"{br[0]}:ACCESS_LOCKBOX.md", cwd=bare).stdout
|
||||
assert f"- **`NEW_TEST_KEY`**: `{FAKE}`" in content and "EXISTING_KEY" in content
|
||||
# main is untouched
|
||||
assert FAKE not in sh("git", "show", "main:ACCESS_LOCKBOX.md", cwd=bare).stdout
|
||||
|
||||
|
||||
def test_refuses_existing_key_both_formats_and_bad_input(tmp_path):
|
||||
bare = make_remote(tmp_path)
|
||||
for k in ("EXISTING_KEY", "OLD_ENV_KEY"):
|
||||
rc, out = put(tmp_path, bare, k, FAKE)
|
||||
assert rc == 3 and "already exists" in out and FAKE not in out
|
||||
assert put(tmp_path, bare, "lower_case", FAKE)[0] == 2
|
||||
assert put(tmp_path, bare, "OK_KEY_1", FAKE, mode=0o644)[0] == 2 # not 0600
|
||||
assert put(tmp_path, bare, "OK_KEY_2", "has space `tick`")[0] == 2 # unsafe value
|
||||
assert not sh("git", "branch", "--list", "lockbox-put/*", cwd=bare).stdout.strip() # nothing pushed
|
||||
|
||||
|
||||
def test_symlink_refused(tmp_path):
|
||||
bare = make_remote(tmp_path)
|
||||
real = tmp_path / "real"
|
||||
real.write_text(FAKE)
|
||||
os.chmod(real, 0o600)
|
||||
link = tmp_path / "link"
|
||||
link.symlink_to(real)
|
||||
e = {**os.environ, "LOCKBOX_PUT_REPO": str(bare), "LOCKBOX_PUT_NO_PR": "1"}
|
||||
r = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_put.py"), "SYM_KEY", str(link)],
|
||||
capture_output=True, text=True, env=e)
|
||||
assert r.returncode == 2 and FAKE not in r.stdout + r.stderr
|
||||
@@ -117,3 +117,40 @@ def test_public_scan_excuses_by_hash_and_by_path():
|
||||
# a PEM header anywhere outside the allowed paths still counts
|
||||
assert not ps.excused("windy-agent", "private key block", "ffff0000", ["tests/keys/a.pem", "deploy/k.pem"], allow)
|
||||
assert not ps.excused("other", "openai key", "aaaa1111", ["x"], allow)
|
||||
|
||||
|
||||
HEX32 = "0123456789abcdef" * 2 # synthetic
|
||||
|
||||
|
||||
def test_twilio_shapes_hash_only_and_no_md5_noise():
|
||||
kinds = lambda t: [k for k, _ in ss.find(t)] # noqa: E731
|
||||
assert kinds(f'TWILIO_AUTH_TOKEN = "{HEX32}"') == ["32-hex secret assignment"]
|
||||
assert kinds(f"auth_token: {HEX32}") == ["32-hex secret assignment"]
|
||||
assert kinds("AC" + HEX32) == ["twilio sid/api key"]
|
||||
assert kinds("SK" + HEX32) == ["twilio sid/api key"]
|
||||
# plain md5 / uuid-without-dashes / a 64-hex sha256 are NOT secrets by shape
|
||||
assert kinds(f"md5 = {HEX32}") == []
|
||||
assert kinds(f"checksum_key = {HEX32}{HEX32}") == []
|
||||
assert kinds(f"name = 'x{HEX32}'") == []
|
||||
# the hash is of the value alone, so renaming the variable keeps the same allow hash
|
||||
a = ss.find(f"A_TOKEN={HEX32}")[0][1]
|
||||
b = ss.find(f"OTHER_SECRET: '{HEX32}'")[0][1]
|
||||
assert a == b == ss.h8(HEX32)
|
||||
assert HEX32 not in repr(ss.find(f"A_TOKEN={HEX32}"))
|
||||
|
||||
|
||||
def test_warn_kinds_do_not_block(monkeypatch):
|
||||
f = sg.cg.Finding("a.py", 1, "32-hex secret assignment", "32-hex secret assignment #abcd1234")
|
||||
monkeypatch.setattr(sg, "MODE", "block")
|
||||
monkeypatch.setattr(sg, "WARN_KINDS", {"32-hex secret assignment"})
|
||||
assert sg.status_for([f], True)[0] == "success"
|
||||
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
||||
assert sg.status_for([f], True)[0] == "failure"
|
||||
|
||||
|
||||
def test_pypi_token_shape_hash_only():
|
||||
tok = "pypi-AgE" + "Ab1_-" * 20 # synthetic
|
||||
got = ss.find(f"password = {tok}")
|
||||
assert [k for k, _ in got] == ["pypi token"] and got[0][1] == ss.h8(tok)
|
||||
assert tok not in repr(got)
|
||||
assert ss.find("pypi-AgE-too-short") == []
|
||||
|
||||
106
api/tests/test_secret_scan.py
Normal file
106
api/tests/test_secret_scan.py
Normal file
@@ -0,0 +1,106 @@
|
||||
"""secret-scan + env-names: findings carry label/location/hash, NEVER a value or fragment."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
# Synthetic, random-looking, never real. FAKE_LB is in the fake lockbox; TWI matches a shape.
|
||||
FAKE_LB = "k7Xv2QpLm9RtZw4HnB8dYc3S"
|
||||
TWI = "9f3a7c1e5b2d48806a1f4e7d2c9b0835"
|
||||
|
||||
|
||||
def run(script, *args, env=None):
|
||||
e = {**os.environ, **(env or {})}
|
||||
r = subprocess.run([sys.executable, str(SCRIPTS / script), *args], capture_output=True, text=True, env=e)
|
||||
return r.returncode, r.stdout + r.stderr
|
||||
|
||||
|
||||
def no_fragment(out: str, value: str, n: int = 6):
|
||||
assert value not in out
|
||||
for i in range(len(value) - n + 1):
|
||||
assert value[i:i + n] not in out, f"fragment of the value leaked at {i}"
|
||||
|
||||
|
||||
def seeded(tmp_path):
|
||||
lb = tmp_path / "lockbox.md"
|
||||
lb.write_text(f"FAKE_VENDOR_API_KEY={FAKE_LB}\nNOTE: nothing here\n")
|
||||
repo = tmp_path / "repo"
|
||||
repo.mkdir()
|
||||
g = lambda *a: subprocess.run(["git", "-C", str(repo), *a], check=True, capture_output=True) # noqa: E731
|
||||
g("init", "-q", "-b", "main")
|
||||
g("config", "user.email", "t@t")
|
||||
g("config", "user.name", "t")
|
||||
(repo / "app.py").write_text(f'KEY = "{FAKE_LB}"\nTWILIO_AUTH_TOKEN = "{TWI}"\n')
|
||||
g("add", "-A")
|
||||
g("commit", "-qm", "add secrets")
|
||||
(repo / "app.py").write_text("KEY = None\n") # removed from HEAD, still in history
|
||||
g("commit", "-qam", "remove")
|
||||
return lb, repo
|
||||
|
||||
|
||||
def test_tree_scan_labels_locations_no_value(tmp_path):
|
||||
lb, repo = seeded(tmp_path)
|
||||
(repo / "live.py").write_text(f'x = "{FAKE_LB}"\n')
|
||||
rc, out = run("secret_scan.py", str(repo / "live.py"), env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb)})
|
||||
assert rc == 1
|
||||
assert "live.py:1" in out and "lockbox:FAKE_VENDOR_API_KEY" in out
|
||||
no_fragment(out, FAKE_LB)
|
||||
|
||||
|
||||
def test_history_finds_removed_secret_with_commit(tmp_path):
|
||||
lb, repo = seeded(tmp_path)
|
||||
rc, out = run("secret_scan.py", str(repo), "--history", env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb)})
|
||||
assert rc == 1
|
||||
assert "app.py:1" in out and "commit=" in out and "lockbox:FAKE_VENDOR_API_KEY" in out
|
||||
assert "app.py:2" in out and "32-hex secret assignment" in out
|
||||
no_fragment(out, FAKE_LB)
|
||||
no_fragment(out, TWI)
|
||||
|
||||
|
||||
def test_repo_flag_clones_scans_and_cleans_up(tmp_path):
|
||||
lb, repo = seeded(tmp_path)
|
||||
cache = tmp_path / "home"
|
||||
(cache / ".cache").mkdir(parents=True)
|
||||
rc, out = run("secret_scan.py", "--repo", str(repo),
|
||||
env={"SECRET_SCAN_LOCKBOX_PATHS": str(lb), "HOME": str(cache)})
|
||||
assert rc == 1 and "app.py:1" in out
|
||||
no_fragment(out, FAKE_LB)
|
||||
assert not [p for p in (cache / ".cache").iterdir() if p.name.startswith("secret-scan-")]
|
||||
|
||||
|
||||
def test_clean_tree_and_bad_input(tmp_path):
|
||||
(tmp_path / "ok.txt").write_text("hello world\n")
|
||||
rc, out = run("secret_scan.py", str(tmp_path / "ok.txt"), "--no-lockbox")
|
||||
assert rc == 0 and "0 finding(s)" in out
|
||||
rc, out = run("secret_scan.py", str(tmp_path), "--history", "--no-lockbox")
|
||||
assert rc == 2 and "needs a git repo" in out
|
||||
|
||||
|
||||
def test_env_names_never_prints_values(tmp_path):
|
||||
a = tmp_path / "a.env"
|
||||
b = tmp_path / "b.env"
|
||||
a.write_text(f"# c\nexport DB_PASSWORD={FAKE_LB}\nTOKEN=\"{TWI}\"\nEMPTY=\nONLY_A=1\n")
|
||||
b.write_text(f"DB_PASSWORD={FAKE_LB}\nTOKEN=different-value-here\nONLY_B=2\n")
|
||||
rc, out = run("env_names.py", str(a), "--hash")
|
||||
assert rc == 0 and "DB_PASSWORD" in out and "set" in out and "empty" in out and "len=24" in out
|
||||
assert "sha256:" in out
|
||||
no_fragment(out, FAKE_LB)
|
||||
no_fragment(out, TWI, 7)
|
||||
rc, out = run("env_names.py", str(a), "--compare", str(b))
|
||||
assert "DB_PASSWORD" in out and "SAME" in out and "DIFFERENT" in out
|
||||
assert "only-in-A" in out and "only-in-B" in out
|
||||
no_fragment(out, FAKE_LB)
|
||||
rc, out = run("env_names.py", str(tmp_path / "missing.env"))
|
||||
assert rc == 2
|
||||
|
||||
|
||||
def test_shapes_are_the_guards_shapes():
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
import secret_scan as sc
|
||||
import secret_shapes as ss
|
||||
assert sc.ss is ss # one source of shapes: a new guard shape is automatically a scan shape
|
||||
@@ -45,3 +45,8 @@ allow:
|
||||
- repo: windy-git
|
||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||
reason: "The guard's own pattern list and this file."
|
||||
|
||||
- repo: windy-mind
|
||||
paths: ["*"]
|
||||
matches: [':11434']
|
||||
reason: "Windy Mind IS the compute door (endpoint + key); it may call Ollama. Only the Ollama port is allowed here, any provider host/SDK in Mind still flags."
|
||||
|
||||
@@ -51,3 +51,9 @@ allow:
|
||||
paths: ["api/tests/test_secret_guard.py"]
|
||||
kinds: [private key block]
|
||||
reason: "The guard's own test uses a PEM header string as a sample."
|
||||
- repo: windy-code
|
||||
hashes: ["23f32607"]
|
||||
reason: "VS Code OSS extensions' package.json aiKey: Microsoft's public telemetry (App Insights) key, shipped in every VS Code build; not a Windy credential."
|
||||
- repo: windytalk
|
||||
hashes: ["c4189d79"]
|
||||
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
||||
|
||||
14
deploy/systemd/windygit-state-backup.service
Normal file
14
deploy/systemd/windygit-state-backup.service
Normal file
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2)
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/windygit/src
|
||||
# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass
|
||||
# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD).
|
||||
EnvironmentFile=/srv/windygit/src/.env
|
||||
ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=3h
|
||||
@@ -0,0 +1,3 @@
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||
11
deploy/systemd/windygit-state-backup.timer
Normal file
11
deploy/systemd/windygit-state-backup.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Nightly Windy Git state backup
|
||||
|
||||
[Timer]
|
||||
# 03:07 local, clear of the git-bundle backup at 04:17.
|
||||
OnCalendar=*-*-* 03:07:00
|
||||
Persistent=true
|
||||
RandomizedDelaySec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -32,7 +32,7 @@ services:
|
||||
|
||||
gitea:
|
||||
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
||||
image: docker.io/gitea/gitea:1.24.6
|
||||
image: docker.io/gitea/gitea:1.24.7
|
||||
environment:
|
||||
GITEA__database__DB_TYPE: postgres
|
||||
GITEA__database__HOST: db:5432
|
||||
|
||||
33
docs/RESTORE-DRILL.md
Normal file
33
docs/RESTORE-DRILL.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Restoring Windy Git from the encrypted state backup
|
||||
|
||||
What is backed up (`scripts/backup_state.sh`, restic repo `s3:…/windy-git-backups/restic`, tag `windygit-state`):
|
||||
both Postgres databases (`gitea`, `windygit`, custom-format dumps + globals), the whole Gitea data root
|
||||
(`/srv/windygit/git`: config, jwt, attachments, avatars, templates AND the bare repositories),
|
||||
`/srv/windygit/src/.env`, `deploy/runner/.env`, `/etc/cloudflared`, the windygit systemd drop-ins.
|
||||
NOT in it: the restic password itself (lockbox `RESTIC_WINDYGIT_PASSWORD`) and the R2 access key
|
||||
(scoped token `windy-git-r2-scoped`, lockbox). Never print either: use `lockbox-get KEY FILE`.
|
||||
|
||||
## Drill (any machine with restic + docker; proven on Veron 2026-10-01, counts identical)
|
||||
export RESTIC_PASSWORD_FILE=<0600 file from lockbox-get RESTIC_WINDYGIT_PASSWORD>
|
||||
export AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… # from lockbox-get, into env, not echoed
|
||||
export RESTIC_REPOSITORY=s3:https://<R2 account>.r2.cloudflarestorage.com/windy-git-backups/restic
|
||||
restic snapshots --tag windygit-state
|
||||
restic restore latest --tag windygit-state --target /var/tmp/wg-drill
|
||||
docker run -d --name wg-drill-pg -e POSTGRES_PASSWORD=<random> -e POSTGRES_USER=drill postgres:16-alpine
|
||||
for db in gitea windygit; do
|
||||
docker exec wg-drill-pg psql -U drill -d postgres -c "create database $db"
|
||||
docker exec -i wg-drill-pg pg_restore -U drill -d $db --no-owner --no-privileges \
|
||||
< /var/tmp/wg-drill/var/backups/windygit-state/$db.dump
|
||||
done
|
||||
# compare row counts with live (or with the last known): repository, issue, pull_request, "user",
|
||||
# external_login_user, access_token, action_run, action_run_job
|
||||
docker rm -f wg-drill-pg; rm -rf /var/tmp/wg-drill
|
||||
|
||||
## Real disaster (Veron lost)
|
||||
1. New Linux host with Docker, a Cloudflare tunnel connector, the repo (`git clone` from GitHub: windy-git).
|
||||
2. `restic restore latest --tag windygit-state --target /` (puts /srv/windygit/git, the .env files, /etc/cloudflared back).
|
||||
3. `docker compose -p windy-git up -d db`, then pg_restore both dumps into it (as above, into the real db names/owner from `.env`).
|
||||
4. `docker compose -p windy-git up -d` + `deploy/runner` runners; re-register runners if the token changed.
|
||||
5. Verify: `/api/healthz`, Windy SSO login, `git ls-remote`, one CI run. GitHub is still the source of truth for code,
|
||||
so repo content can also be re-synced from there; the database is what only this backup holds.
|
||||
Retention: 14 daily / 8 weekly / 6 monthly (prune on Sundays). Integrity: every run does `restic check --read-data-subset=2%`.
|
||||
66
scripts/backup_state.sh
Executable file
66
scripts/backup_state.sh
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly STATE backup: everything git bundles do NOT hold (SOTU 10-01: 625 issues/PRs, users,
|
||||
# SSO links, CI history, settings lived on one unbacked-up host). Encrypted restic repo in R2.
|
||||
# - Postgres: every database (custom-format dump, restore-listable) + globals
|
||||
# - Gitea config/data (app.ini, jwt, attachments, avatars, templates) + the bare repositories
|
||||
# - the deploy .env files, systemd drop-ins and the cloudflared tunnel config (needed to rebuild)
|
||||
# The restic password lives in /etc/windygit/restic.pass (root 600) AND the lockbox
|
||||
# (RESTIC_WINDYGIT_PASSWORD): a lost Veron must not lose the backups. NEVER echo env/values here.
|
||||
# Restore: docs/RESTORE-DRILL.md. Bounded: every docker exec runs under `timeout` (a hung
|
||||
# runc exec in the IO stall wedged the sync on 09-23).
|
||||
set -euo pipefail
|
||||
log() { echo "[backup_state $(date -u +%FT%TZ)] $*"; }
|
||||
: "${R2_ACCOUNT_ID:?}" "${R2_ACCESS_KEY_ID:?}" "${R2_SECRET_ACCESS_KEY:?}"
|
||||
PASSFILE="${RESTIC_PASSWORD_FILE:-/etc/windygit/restic.pass}"
|
||||
[[ -s "$PASSFILE" ]] || { log "FATAL: $PASSFILE missing/empty: refusing to report a backup that did not happen"; exit 1; }
|
||||
export RESTIC_PASSWORD_FILE="$PASSFILE"
|
||||
export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY"
|
||||
export RESTIC_REPOSITORY="${RESTIC_REPOSITORY:-s3:https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com/${R2_BUCKET_BACKUPS:-windy-git-backups}/restic}"
|
||||
DB="${WG_DB_CONTAINER:-windy-git-db-1}"
|
||||
STAGE="${WG_STAGE:-/var/backups/windygit-state}"
|
||||
GIT_ROOT="${GIT_DATA_ROOT:-/srv/windygit/git}"
|
||||
umask 077
|
||||
mkdir -p "$STAGE"; chmod 700 "$STAGE"; rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||
|
||||
# systemd gives units no $HOME, and restic wants a cache dir: pin one.
|
||||
export RESTIC_CACHE_DIR="${RESTIC_CACHE_DIR:-/var/cache/windygit-restic}"; mkdir -p "$RESTIC_CACHE_DIR"
|
||||
if ! err=$(restic cat config 2>&1 >/dev/null); then
|
||||
# only a MISSING repo may be initialised; any other error (auth, network, wrong password) must stop here
|
||||
if grep -qiE "does not exist|is there a repository|unable to open config file" <<<"$err"; then
|
||||
log "initialising restic repo"; restic init >/dev/null
|
||||
else
|
||||
log "FATAL: restic cannot open the repository: $(head -c 300 <<<"$err" | tr '\n' ' ')"; exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
PGU=$(timeout 30 docker exec "$DB" printenv POSTGRES_USER)
|
||||
[[ -n "$PGU" ]] || { log "FATAL: no POSTGRES_USER in $DB"; exit 1; }
|
||||
dbs=$(timeout 60 docker exec "$DB" psql -U "$PGU" -Atc "select datname from pg_database where not datistemplate and datname<>'postgres' order by 1")
|
||||
n=0
|
||||
for d in $dbs; do
|
||||
timeout 600 docker exec "$DB" pg_dump -U "$PGU" -Fc "$d" > "$STAGE/$d.dump"
|
||||
# a dump that cannot be listed is not a backup
|
||||
timeout 120 docker exec -i "$DB" pg_restore -l < "$STAGE/$d.dump" >/dev/null
|
||||
[[ $(stat -c%s "$STAGE/$d.dump") -gt 1000 ]] || { log "FATAL: $d dump suspiciously small"; exit 1; }
|
||||
n=$((n+1)); log "dumped $d ($(stat -c%s "$STAGE/$d.dump") bytes)"
|
||||
done
|
||||
[[ $n -ge 1 ]] || { log "FATAL: no databases dumped"; exit 1; }
|
||||
timeout 120 docker exec "$DB" pg_dumpall -U "$PGU" --globals-only > "$STAGE/globals.sql"
|
||||
|
||||
paths=("$STAGE" "$GIT_ROOT" /srv/windygit/src/.env /srv/windygit/src/deploy/runner/.env /etc/cloudflared)
|
||||
for p in /etc/systemd/system/windygit-*.service.d /etc/windygit; do [[ -e $p ]] && paths+=("$p"); done
|
||||
# restic.pass itself is excluded: the password never rides in its own backup
|
||||
snap=$(restic backup --tag windygit-state --host windygit-veron --quiet --json \
|
||||
--exclude "$GIT_ROOT/gitea/log" --exclude "$GIT_ROOT/gitea/queues" --exclude "$GIT_ROOT/gitea/tmp" \
|
||||
--exclude "$GIT_ROOT/gitea/indexers" --exclude "$GIT_ROOT/gitea/actions_log" --exclude /etc/windygit/restic.pass \
|
||||
"${paths[@]}" | python3 -c 'import sys,json
|
||||
for l in sys.stdin:
|
||||
d=json.loads(l)
|
||||
if d.get("message_type")=="summary": print(d["snapshot_id"][:8])')
|
||||
[[ -n "$snap" ]] || { log "FATAL: restic produced no snapshot"; exit 1; }
|
||||
rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||
restic check --read-data-subset=2% --quiet >/dev/null || { log "FATAL: restic check failed"; exit 1; }
|
||||
if [[ $(date +%u) == 7 ]]; then
|
||||
restic forget --tag windygit-state --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --prune --quiet >/dev/null
|
||||
fi
|
||||
echo "ok — state backed up ($n dbs, snapshot $snap)"
|
||||
@@ -63,6 +63,9 @@ JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/gen
|
||||
|
||||
RULES: list[tuple[str, re.Pattern]] = [
|
||||
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||
# Grant via Boss 10-01: compute = Windy Mind. A NEW reference to an Ollama port (Veron's :11434) is a
|
||||
# direct call around Mind's metering/caps. WARN-only, never red, and only for lines a PR ADDS.
|
||||
("veron ollama", re.compile(r"(?::|%3[aA])11434(?![0-9])")),
|
||||
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||
@@ -70,6 +73,9 @@ RULES: list[tuple[str, re.Pattern]] = [
|
||||
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||
]
|
||||
# Kinds that never block (even in MODE=block) and are only judged on ADDED lines, never the baseline tree.
|
||||
WARN_ONLY_KINDS = {"veron ollama"}
|
||||
OLLAMA_MSG = "compute = Windy Mind (endpoint + key); do not call Veron's Ollama directly"
|
||||
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||
|
||||
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||
@@ -253,7 +259,8 @@ def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> li
|
||||
allow = load_allow()
|
||||
fp = _fingerprint(allow)
|
||||
if is_default_head:
|
||||
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
||||
return cached_scan(f"tree:{repo}:{sha}:{fp}",
|
||||
lambda: [f for f in scan_tree(repo, bare, sha, allow) if f.kind not in WARN_ONLY_KINDS])
|
||||
return cached_scan(
|
||||
f"pr:{repo}:{sha}:{fp}",
|
||||
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||
@@ -268,6 +275,11 @@ def status_for(findings: list[Finding], whole_tree: bool,
|
||||
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
||||
09-23: his desktop work is never blocked by us)."""
|
||||
scope = "in tree" if whole_tree else "added"
|
||||
soft = [f for f in findings if f.kind in WARN_ONLY_KINDS]
|
||||
findings = [f for f in findings if f.kind not in WARN_ONLY_KINDS]
|
||||
if not findings and not grant and soft:
|
||||
f = soft[0]
|
||||
return "success", f"⚠ WARN: new Veron Ollama ref {f.path}:{f.line}. {OLLAMA_MSG}"[:140], f
|
||||
if not findings and grant:
|
||||
g, n = grant[0], len(grant)
|
||||
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
||||
|
||||
23
scripts/drill_cross_host.sh
Executable file
23
scripts/drill_cross_host.sh
Executable file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed.
|
||||
# Usage: bash drill_cross_host.sh (needs lockbox keys RESTIC_WINDYGIT_PASSWORD, WINDYGIT_R2_ACCESS_KEY_ID, WINDYGIT_R2_SECRET_ACCESS_KEY, WINDYGIT_R2_ENDPOINT)
|
||||
set -euo pipefail
|
||||
umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX)
|
||||
trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT
|
||||
lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null
|
||||
lockbox-get WINDYGIT_R2_ACCESS_KEY_ID "$W/ak" >/dev/null; lockbox-get WINDYGIT_R2_SECRET_ACCESS_KEY "$W/sk" >/dev/null; lockbox-get WINDYGIT_R2_ENDPOINT "$W/ep" >/dev/null
|
||||
export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")"
|
||||
export RESTIC_REPOSITORY="s3:$(cat "$W/ep")/windy-git-backups/restic"
|
||||
restic snapshots --tag windygit-state --compact | tail -3
|
||||
restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet
|
||||
ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}'
|
||||
docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null
|
||||
for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done
|
||||
for db in gitea windygit; do
|
||||
docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db"
|
||||
docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true
|
||||
done
|
||||
for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do
|
||||
echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")"
|
||||
done
|
||||
echo "cross-host drill OK (cleaned up)"
|
||||
153
scripts/env_names.py
Normal file
153
scripts/env_names.py
Normal file
@@ -0,0 +1,153 @@
|
||||
#!/usr/bin/env python3
|
||||
"""env-names: list environment variable NAMES only (Boss ruling 10-01, house rule 10).
|
||||
|
||||
env-names <docker container | systemd unit | env file> [--host H] [--sudo] [--hash]
|
||||
env-names A --compare B [--host H] [--host2 H2]
|
||||
|
||||
Prints NAME, set|empty, and value LENGTH. Never a value or fragment. --hash adds sha256[:8]
|
||||
(compare two places for equality; a hash of a weak value can be guessed, so use it for
|
||||
real secrets only). --compare prints SAME / DIFFERENT / only-in-A / only-in-B per name
|
||||
(equality by full-value hash, nothing else shown). Values live in memory only.
|
||||
Targets: an existing file (dotenv style) | a docker container name | a systemd unit
|
||||
(Environment= + EnvironmentFile=; --sudo to read root-only files). --host runs the docker /
|
||||
systemctl / file read over ssh (alias from ~/.ssh/config).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
KV = ("=",)
|
||||
|
||||
|
||||
def run(cmd: list[str], host: str | None, sudo: bool = False) -> tuple[int, str]:
|
||||
if sudo:
|
||||
cmd = ["sudo", "-n", *cmd]
|
||||
if host:
|
||||
cmd = ["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", host, shlex.join(cmd)]
|
||||
r = subprocess.run(cmd, capture_output=True, text=True, errors="ignore", timeout=60)
|
||||
return r.returncode, r.stdout
|
||||
|
||||
|
||||
def parse_dotenv(text: str) -> dict[str, str]:
|
||||
out: dict[str, str] = {}
|
||||
for raw in text.splitlines():
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
if line.startswith("export "):
|
||||
line = line[7:].lstrip()
|
||||
k, v = line.split("=", 1)
|
||||
k = k.strip()
|
||||
v = v.strip()
|
||||
if len(v) >= 2 and v[0] == v[-1] and v[0] in "\"'":
|
||||
v = v[1:-1]
|
||||
if k.replace("_", "").isalnum() and not k[0].isdigit():
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
def from_file(path: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||
if host or sudo:
|
||||
rc, out = run(["cat", path], host, sudo)
|
||||
return parse_dotenv(out) if rc == 0 else None
|
||||
try:
|
||||
with open(path, errors="ignore") as fh:
|
||||
return parse_dotenv(fh.read())
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def from_docker(name: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||
rc, out = run(["docker", "inspect", "-f", "{{json .Config.Env}}", name], host, sudo)
|
||||
if rc != 0 or not out.strip():
|
||||
return None
|
||||
try:
|
||||
items = json.loads(out)
|
||||
except ValueError:
|
||||
return None
|
||||
return {k: v for k, _, v in (i.partition("=") for i in (items or []))}
|
||||
|
||||
|
||||
def from_systemd(unit: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||
rc, out = run(["systemctl", "show", unit, "-p", "Environment", "-p", "EnvironmentFiles"], host)
|
||||
if rc != 0 or "LoadState=not-found" in out:
|
||||
return None
|
||||
env: dict[str, str] = {}
|
||||
files: list[str] = []
|
||||
for line in out.splitlines():
|
||||
if line.startswith("Environment="):
|
||||
for tok in shlex.split(line[len("Environment="):]):
|
||||
k, _, v = tok.partition("=")
|
||||
env[k] = v
|
||||
elif line.startswith("EnvironmentFiles="):
|
||||
f = line[len("EnvironmentFiles="):].split(" (")[0].strip().lstrip("-")
|
||||
if f:
|
||||
files.append(f)
|
||||
for f in files: # later files override earlier, like systemd
|
||||
d = from_file(f, host, sudo)
|
||||
if d is None:
|
||||
print(f"# note: EnvironmentFile {f} unreadable (try --sudo)", file=sys.stderr)
|
||||
else:
|
||||
env.update(d)
|
||||
return env
|
||||
|
||||
|
||||
def load(target: str, host: str | None, sudo: bool) -> dict[str, str] | None:
|
||||
if (not host and os.path.isfile(target)) or target.startswith(("/", "./", "~")):
|
||||
return from_file(os.path.expanduser(target), host, sudo)
|
||||
if target.endswith((".service", ".timer", ".socket")):
|
||||
return from_systemd(target, host, sudo)
|
||||
return from_docker(target, host, sudo) or from_systemd(target, host, sudo)
|
||||
|
||||
|
||||
def sh(v: str) -> str:
|
||||
return hashlib.sha256(v.encode()).hexdigest()
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="env-names", description="env var NAMES only")
|
||||
ap.add_argument("target")
|
||||
ap.add_argument("--host")
|
||||
ap.add_argument("--host2", help="ssh host for the --compare target")
|
||||
ap.add_argument("--sudo", action="store_true")
|
||||
ap.add_argument("--hash", action="store_true", help="add sha256[:8] per variable")
|
||||
ap.add_argument("--compare", metavar="TARGET2")
|
||||
a = ap.parse_args(argv)
|
||||
env = load(a.target, a.host, a.sudo)
|
||||
if env is None:
|
||||
print(f"error: could not read {a.target!r} (file, docker container or systemd unit)")
|
||||
return 2
|
||||
if a.compare:
|
||||
env2 = load(a.compare, a.host2 or a.host, a.sudo)
|
||||
if env2 is None:
|
||||
print(f"error: could not read {a.compare!r}")
|
||||
return 2
|
||||
for k in sorted(set(env) | set(env2)):
|
||||
if k not in env2:
|
||||
print(f"{k:<40} only-in-A")
|
||||
elif k not in env:
|
||||
print(f"{k:<40} only-in-B")
|
||||
else:
|
||||
print(f"{k:<40} {'SAME' if sh(env[k]) == sh(env2[k]) else 'DIFFERENT'}"
|
||||
f" (len {len(env[k])} vs {len(env2[k])})")
|
||||
return 0
|
||||
for k in sorted(env):
|
||||
v = env[k]
|
||||
extra = f" sha256:{sh(v)[:8]}" if a.hash and v else ""
|
||||
print(f"{k:<40} {'set ' if v else 'empty'} len={len(v)}{extra}")
|
||||
print(f"# {len(env)} variable(s); values never printed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except Exception as e: # never a traceback
|
||||
print(f"error: {type(e).__name__}")
|
||||
sys.exit(2)
|
||||
14
scripts/install_secret_tools.sh
Executable file
14
scripts/install_secret_tools.sh
Executable file
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Install the shared hash-only secret tools on THIS machine (Windy 0): secret-scan + env-names.
|
||||
# Source of truth is this repo (scripts/); re-run after a pull to update.
|
||||
set -euo pipefail
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
dest="$HOME/.local/share/secret-tools"
|
||||
mkdir -p "$dest" "$HOME/.local/bin"
|
||||
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$dest/"
|
||||
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py"; do
|
||||
n=${pair%%:*}; f=${pair##*:}
|
||||
printf '#!/usr/bin/env bash\nexec python3 "%s/%s" "$@"\n' "$dest" "$f" > "$HOME/.local/bin/$n"
|
||||
chmod 755 "$HOME/.local/bin/$n"
|
||||
done
|
||||
echo "installed secret-scan, env-names and lockbox-put (shapes from secret_shapes.py, same as secret-guard)"
|
||||
141
scripts/lockbox_put.py
Normal file
141
scripts/lockbox_put.py
Normal file
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lockbox-put: add ONE secret to the lockbox by PR, without anyone reading, printing or
|
||||
grepping the lockbox (Boss rule 10-01; Windy Hub ruling).
|
||||
|
||||
lockbox-put KEY FILE [--lane NAME] [--note TEXT]
|
||||
|
||||
KEY exact name, ^[A-Z][A-Z0-9_]{2,63}$ . FILE a 0600 file you own (not a symlink) whose
|
||||
content is the value (one line). Appends ONE line `- **`KEY`**: `<value>`` (the format
|
||||
lockbox-get reads) under a new heading at the END of ACCESS_LOCKBOX.md in a fresh temp clone,
|
||||
on a new branch, and opens a kit-army-config PR. Append-only: the diff is verified to be one
|
||||
file, additions only, before pushing. REFUSES if KEY already exists (a bool computed in
|
||||
memory; no line, value or location is ever printed). Reviewers see the KEY NAME + lane only
|
||||
if they look at the diff; the PR body never carries the value. Never echoes the value.
|
||||
Env (tests): LOCKBOX_PUT_REPO=<clone url/path>, LOCKBOX_PUT_NO_PR=1.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
REPO = os.environ.get("LOCKBOX_PUT_REPO", "https://github.com/sneakyfree/kit-army-config.git")
|
||||
SLUG = "sneakyfree/kit-army-config"
|
||||
KEY_RE = re.compile(r"^[A-Z][A-Z0-9_]{2,63}$")
|
||||
VAL_RE = re.compile(r"^[A-Za-z0-9._~+/=:@%,-]{8,512}$") # no backtick, quote, space or newline
|
||||
|
||||
|
||||
def die(msg: str, code: int = 2):
|
||||
print(f"lockbox-put: {msg}")
|
||||
sys.exit(code)
|
||||
|
||||
|
||||
def git(cwd: str, *a: str, quiet=True) -> subprocess.CompletedProcess:
|
||||
# stderr is dropped: git/gh errors can echo URLs; stdout only when we need it.
|
||||
return subprocess.run(["git", "-C", cwd, *a], capture_output=True, text=True, errors="ignore")
|
||||
|
||||
|
||||
def key_exists(clone: str, key: str) -> bool:
|
||||
"""True if KEY is already defined anywhere lockbox-get reads. Bool only, nothing printed."""
|
||||
pat_env = re.compile(r"^" + re.escape(key) + r"=")
|
||||
pat_md = re.compile(r"^\s*[-*]?\s*\*\*`" + re.escape(key) + r"`\*\*\s*:")
|
||||
paths = [Path(clone, "ACCESS_LOCKBOX.md")] + [
|
||||
Path(dp, f) for dp, _d, fs in os.walk(Path(clone, "secrets")) for f in fs if f.endswith(".env")]
|
||||
for p in paths:
|
||||
try:
|
||||
with open(p, errors="ignore") as fh:
|
||||
for line in fh:
|
||||
if pat_env.match(line) or pat_md.match(line):
|
||||
return True
|
||||
except OSError:
|
||||
continue
|
||||
return False
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="lockbox-put")
|
||||
ap.add_argument("key")
|
||||
ap.add_argument("file")
|
||||
ap.add_argument("--lane", default=os.environ.get("LOCKBOX_LANE", "a lane"))
|
||||
ap.add_argument("--note", default="")
|
||||
a = ap.parse_args(argv)
|
||||
if not KEY_RE.match(a.key):
|
||||
die("KEY must match ^[A-Z][A-Z0-9_]{2,63}$")
|
||||
try:
|
||||
st = os.lstat(a.file)
|
||||
except OSError:
|
||||
die("FILE not found")
|
||||
if stat.S_ISLNK(st.st_mode) or not stat.S_ISREG(st.st_mode):
|
||||
die("FILE must be a regular file (not a symlink)")
|
||||
if st.st_uid != os.getuid() or (st.st_mode & 0o077):
|
||||
die("FILE must be owned by you and mode 0600")
|
||||
with open(a.file) as fh:
|
||||
value = fh.read().strip()
|
||||
if not VAL_RE.match(value):
|
||||
die("value must be one line of 8-512 chars from [A-Za-z0-9._~+/=:@%,-] (no spaces, quotes, backticks)")
|
||||
note = re.sub(r"[`\n\r]", " ", a.note)[:160]
|
||||
lane = re.sub(r"[^A-Za-z0-9 ._-]", "", a.lane)[:40]
|
||||
tmp = tempfile.mkdtemp(prefix="lockbox-put-", dir=str(Path.home() / ".cache") if (Path.home() / ".cache").is_dir() else None)
|
||||
os.chmod(tmp, 0o700)
|
||||
clone = os.path.join(tmp, "k")
|
||||
try:
|
||||
if subprocess.run(["git", "clone", "-q", "--depth", "1", REPO, clone],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode != 0:
|
||||
die("clone failed (details withheld: URLs can carry tokens)")
|
||||
if key_exists(clone, a.key):
|
||||
die(f"{a.key} already exists: refusing to overwrite (append-only; pick a new KEY)", 3)
|
||||
lb = Path(clone, "ACCESS_LOCKBOX.md")
|
||||
if not lb.is_file():
|
||||
die("ACCESS_LOCKBOX.md not found in the repo")
|
||||
today = dt.date.today().isoformat()
|
||||
stamp = dt.datetime.now(dt.UTC).strftime("%Y%m%d%H%M")
|
||||
branch = f"lockbox-put/{a.key.lower()}-{stamp}"
|
||||
with open(lb, "a") as fh:
|
||||
fh.write(f"\n## 🗝️ {a.key} (added {today} by {lane} via lockbox-put)\n")
|
||||
fh.write(f"- **`{a.key}`**: `{value}`\n")
|
||||
if note:
|
||||
fh.write(f"- **Note:** {note}\n")
|
||||
git(clone, "checkout", "-q", "-b", branch)
|
||||
git(clone, "add", "ACCESS_LOCKBOX.md")
|
||||
ns = git(clone, "diff", "--cached", "--numstat").stdout.split()
|
||||
# numstat: <added> <deleted> <path>; exactly one file, no deletions
|
||||
if len(ns) != 3 or ns[1] != "0" or ns[2] != "ACCESS_LOCKBOX.md":
|
||||
die("diff is not a pure append to ACCESS_LOCKBOX.md: aborting, nothing pushed")
|
||||
msg = f"lockbox: add {a.key} (via lockbox-put, {lane})\n\nCo-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>"
|
||||
if git(clone, "-c", "user.name=lockbox-put", "-c", "user.email=lockbox-put@windy.invalid",
|
||||
"commit", "-q", "-m", msg).returncode != 0:
|
||||
die("commit failed")
|
||||
if git(clone, "push", "-q", "origin", branch).returncode != 0:
|
||||
die("push failed (details withheld)")
|
||||
if os.environ.get("LOCKBOX_PUT_NO_PR"):
|
||||
print(f"ok: pushed branch {branch} ({a.key}); PR skipped")
|
||||
return 0
|
||||
body = (f"Adds exactly one key: `{a.key}` (by {lane}). Append-only, one file, no deletions "
|
||||
f"(verified before push). Review by KEY NAME only; do not paste the value anywhere.\n\n"
|
||||
f"{note}\n\n🤖 Generated with [Claude Code](https://claude.com/claude-code)")
|
||||
r = subprocess.run(["gh", "pr", "create", "-R", SLUG, "--head", branch, "--base", "main",
|
||||
"--title", f"lockbox: add {a.key} ({lane})", "--body", body],
|
||||
capture_output=True, text=True)
|
||||
if r.returncode != 0:
|
||||
die("branch pushed but `gh pr create` failed; open the PR for the branch by hand")
|
||||
print(f"ok: {a.key} added via PR {r.stdout.strip().splitlines()[-1]}")
|
||||
return 0
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except SystemExit:
|
||||
raise
|
||||
except Exception as e: # never a traceback: it could carry data
|
||||
print(f"lockbox-put: error: {type(e).__name__}")
|
||||
sys.exit(2)
|
||||
@@ -54,7 +54,7 @@ REPOS = os.environ.get(
|
||||
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
||||
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
||||
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
||||
" windy-inbox",
|
||||
" windy-inbox windy-text windy-call windy-cell",
|
||||
).split()
|
||||
|
||||
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
||||
|
||||
@@ -27,6 +27,8 @@ import secret_shapes as ss # noqa: E402
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ALLOW_FILE = Path(os.environ.get("SECRET_GUARD_ALLOW", ROOT / "ci" / "secret-guard-allow.yml"))
|
||||
MODE = os.environ.get("SECRET_GUARD_MODE", "warn")
|
||||
# Kinds that only WARN (rolled out warn-first); empty = every kind blocks in block mode.
|
||||
WARN_KINDS = {k for k in os.environ.get("SECRET_GUARD_WARN_KINDS", "").split(",") if k}
|
||||
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
||||
|
||||
|
||||
@@ -94,8 +96,11 @@ def status_for(findings, whole_tree: bool, grant=()):
|
||||
if not findings:
|
||||
return "success", f"OK: no secret-shaped strings {scope}", None
|
||||
f, n = findings[0], len(findings)
|
||||
state = "failure" if MODE == "block" else "success"
|
||||
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
||||
soft = MODE != "block" or all(x.kind in WARN_KINDS for x in findings)
|
||||
state = "success" if soft else "failure"
|
||||
lead = "⚠ WARN (not blocking)" if soft else "BLOCKED"
|
||||
if not soft:
|
||||
f = next(x for x in findings if x.kind not in WARN_KINDS)
|
||||
return state, f"{lead}: {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
||||
|
||||
|
||||
|
||||
210
scripts/secret_scan.py
Normal file
210
scripts/secret_scan.py
Normal file
@@ -0,0 +1,210 @@
|
||||
#!/usr/bin/env python3
|
||||
"""secret-scan: hash-only secret finder. Boss ruling 10-01 after three lanes printed secrets
|
||||
into their own transcripts while hunting secrets (house rule 10).
|
||||
|
||||
secret-scan <path> [--history] [--repo <git url or path>] [--no-lockbox]
|
||||
|
||||
Reports `file:line` (and the commit with --history), WHICH lockbox entry matched (the KEY
|
||||
NAME only) or which secret SHAPE matched (twilio, zai, aws, ...), plus a sha256[:8] of the
|
||||
token for allow-listing. It NEVER prints, logs or writes a value or any fragment of one
|
||||
(no context line, no masking). The lockbox is loaded in memory only. stdout only.
|
||||
Exit 0 = clean, 1 = findings, 2 = usage/error.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import secret_shapes as ss # noqa: E402 (the SAME shapes as secret-guard)
|
||||
|
||||
HOME = Path.home()
|
||||
LOCKBOX_PATHS = [p for p in os.environ.get("SECRET_SCAN_LOCKBOX_PATHS", "").split(":") if p] or [
|
||||
str(HOME / "kit-army-config" / "secrets"), str(HOME / "kit-army-config" / "ACCESS_LOCKBOX.md")]
|
||||
# Extra shapes that are scan-only (not in the blocking guard): label, regex.
|
||||
EXTRA = [("zai key", re.compile(r"(?<![0-9a-f])[0-9a-f]{32}\.[A-Za-z0-9]{16}(?![A-Za-z0-9])"))]
|
||||
SKIP_DIRS = {".git", "node_modules", "vendor", "third_party", "__pycache__", ".venv"}
|
||||
MAX_BYTES = 5_000_000
|
||||
|
||||
RUN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_\-]{15,199}")
|
||||
UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
|
||||
NAME = re.compile(r"[\s>*`|-]*([A-Za-z][A-Za-z0-9_]{2,60})\s*[=:|]")
|
||||
|
||||
|
||||
def h16(t: str) -> str:
|
||||
return hashlib.sha256(t.encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def cands(line: str):
|
||||
"""Token candidates: runs >=16 chars with a digit and a letter; git shas/uuids excluded."""
|
||||
for chunk in re.split(r"[^A-Za-z0-9_\-.]+", line):
|
||||
parts = [chunk, *chunk.split(".")] if "." in chunk else [chunk]
|
||||
for p in parts:
|
||||
for m in RUN.finditer(p):
|
||||
t = m.group(0)
|
||||
if not (re.search(r"\d", t) and re.search(r"[A-Za-z]", t)):
|
||||
continue
|
||||
if re.fullmatch(r"[0-9a-fA-F]{40}|[0-9a-fA-F]{64}", t) or UUID.match(t.lower()):
|
||||
continue
|
||||
yield t
|
||||
|
||||
|
||||
def load_lockbox() -> dict[str, set[str]]:
|
||||
"""{hash16: {key-name labels}}; values never leave this dict."""
|
||||
out: dict[str, set[str]] = {}
|
||||
files: list[str] = []
|
||||
for p in LOCKBOX_PATHS:
|
||||
if os.path.isdir(p):
|
||||
for root, _d, fs in os.walk(p):
|
||||
files += [os.path.join(root, f) for f in fs]
|
||||
elif os.path.isfile(p):
|
||||
files.append(p)
|
||||
for f in files:
|
||||
try:
|
||||
with open(f, errors="ignore") as fh:
|
||||
for line in fh:
|
||||
m = NAME.match(line)
|
||||
label = m.group(1) if m else "?"
|
||||
for t in cands(line):
|
||||
out.setdefault(h16(t), set()).add(label)
|
||||
except OSError:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def scan_line(line: str, lockbox: dict[str, set[str]]) -> list[tuple[str, str]]:
|
||||
"""[(label, hash8)]: `shape:<kind>` and/or `lockbox:<NAMES>`. No value escapes."""
|
||||
res: list[tuple[str, str]] = []
|
||||
for kind, h in ss.find(line):
|
||||
res.append((f"shape:{kind}", h))
|
||||
for kind, rx in EXTRA:
|
||||
for m in rx.finditer(line):
|
||||
res.append((f"shape:{kind}", ss.h8(m.group(0))))
|
||||
for t in cands(line):
|
||||
k = h16(t)
|
||||
if k in lockbox:
|
||||
names = sorted(n for n in lockbox[k])
|
||||
res.append(("lockbox:" + ",".join(names)[:70], k[:8]))
|
||||
return sorted(set(res))
|
||||
|
||||
|
||||
def is_text(path: Path) -> bool:
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
return b"\0" not in fh.read(4096)
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def scan_tree(root: Path, lockbox):
|
||||
files = [root] if root.is_file() else [
|
||||
Path(dp) / f for dp, dn, fn in os.walk(root) for f in fn
|
||||
if not set(Path(dp).relative_to(root).parts) & SKIP_DIRS]
|
||||
for p in sorted(files):
|
||||
try:
|
||||
if p.stat().st_size > MAX_BYTES or not is_text(p):
|
||||
continue
|
||||
with open(p, errors="ignore") as fh:
|
||||
for n, line in enumerate(fh, 1):
|
||||
for label, h in scan_line(line, lockbox):
|
||||
yield (str(p), n, None, label, h)
|
||||
except OSError:
|
||||
continue
|
||||
|
||||
|
||||
def git(repo: str, *a: str) -> subprocess.Popen:
|
||||
return subprocess.Popen(["git", "--git-dir", repo, *a], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL, text=True, errors="ignore")
|
||||
|
||||
|
||||
def scan_history(gitdir: str, lockbox):
|
||||
"""Every ADDED line on every ref (incl. PR refs). Oldest commit per (hash, file, line)."""
|
||||
seen: dict[tuple, str] = {}
|
||||
p = git(gitdir, "log", "--all", "-p", "-U0", "--no-color", "--format=@@C %h", "-a")
|
||||
commit = path = None
|
||||
ln = 0
|
||||
for row in p.stdout: # type: ignore[union-attr]
|
||||
if row.startswith("@@C "):
|
||||
commit = row[4:].strip()
|
||||
elif row.startswith("+++ "):
|
||||
path = row[6:].strip() if row.startswith("+++ b/") else None
|
||||
elif row.startswith("@@ "):
|
||||
m = re.search(r"\+(\d+)", row)
|
||||
ln = int(m.group(1)) - 1 if m else 0
|
||||
elif row.startswith("+") and path:
|
||||
ln += 1
|
||||
for label, h in scan_line(row[1:], lockbox):
|
||||
seen[(label, h, path, ln)] = commit or "?"
|
||||
p.wait()
|
||||
for (label, h, path, ln), c in sorted(seen.items(), key=lambda x: (x[0][2], x[0][3])):
|
||||
yield (path, ln, c, label, h)
|
||||
|
||||
|
||||
def resolve_gitdir(p: Path) -> str | None:
|
||||
for cand in (p / ".git", p):
|
||||
if (cand / "HEAD").exists() and ((cand / "objects").exists()):
|
||||
return str(cand)
|
||||
return None
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="secret-scan", description=__doc__.split("\n\n")[1] if __doc__ else "")
|
||||
ap.add_argument("path", nargs="?", help="file, directory, or git repo (with --history)")
|
||||
ap.add_argument("--history", action="store_true", help="scan every added line in all git history")
|
||||
ap.add_argument("--repo", help="git URL or path to scan (mirror-cloned to a temp dir, then deleted)")
|
||||
ap.add_argument("--no-lockbox", action="store_true", help="shapes only")
|
||||
a = ap.parse_args(argv)
|
||||
if not (a.path or a.repo):
|
||||
ap.print_usage()
|
||||
return 2
|
||||
lockbox = {} if a.no_lockbox else load_lockbox()
|
||||
print(f"# secret-scan: {len(lockbox)} lockbox tokens in memory, values never printed", flush=True)
|
||||
tmp = None
|
||||
findings = 0
|
||||
try:
|
||||
if a.repo:
|
||||
tmp = tempfile.mkdtemp(prefix="secret-scan-", dir=str(HOME / ".cache") if (HOME / ".cache").is_dir() else None)
|
||||
os.chmod(tmp, 0o700)
|
||||
target = os.path.join(tmp, "r.git")
|
||||
rc = subprocess.run(["git", "clone", "-q", "--mirror", a.repo, target],
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL).returncode
|
||||
if rc != 0:
|
||||
print("error: clone failed (details withheld: URLs can carry tokens)")
|
||||
return 2
|
||||
a.history = True
|
||||
gitdir = target
|
||||
elif a.history:
|
||||
gitdir = resolve_gitdir(Path(a.path))
|
||||
if not gitdir:
|
||||
print("error: --history needs a git repo path")
|
||||
return 2
|
||||
if a.history:
|
||||
for path, ln, c, label, h in scan_history(gitdir, lockbox):
|
||||
findings += 1
|
||||
print(f"{path}:{ln} commit={c} {label} #{h}")
|
||||
else:
|
||||
for path, ln, _c, label, h in scan_tree(Path(a.path), lockbox):
|
||||
findings += 1
|
||||
print(f"{path}:{ln} {label} #{h}")
|
||||
finally:
|
||||
if tmp:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
print(f"# {findings} finding(s)")
|
||||
return 1 if findings else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except KeyboardInterrupt:
|
||||
sys.exit(130)
|
||||
except Exception as e: # never a traceback: it could carry data
|
||||
print(f"error: {type(e).__name__}")
|
||||
sys.exit(2)
|
||||
@@ -18,13 +18,19 @@ PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("openai key", re.compile(r"\bsk-(?:proj-|svcacct-)?(?!ant-)[A-Za-z0-9_-]{32,}")),
|
||||
("stripe live key", re.compile(r"\b[rs]k_live_[A-Za-z0-9]{20,}")),
|
||||
("google api key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])")),
|
||||
# Twilio (Windy Text 10-01: a live auth token sat in test files for months). An auth token
|
||||
# is a bare 32-hex with no prefix, so it is only caught when ASSIGNED to a secret-ish name.
|
||||
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
||||
("32-hex secret assignment", re.compile(
|
||||
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
||||
("pypi token", re.compile(r"\bpypi-AgE[A-Za-z0-9_-]{50,}")),
|
||||
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
||||
]
|
||||
|
||||
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
||||
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
||||
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----")
|
||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}|pypi-AgE")
|
||||
|
||||
|
||||
def h8(value: str | bytes) -> str:
|
||||
@@ -36,5 +42,5 @@ def find(text: str) -> list[tuple[str, str]]:
|
||||
out = []
|
||||
for kind, rx in PATTERNS:
|
||||
for m in rx.finditer(text):
|
||||
out.append((kind, h8(m.group(0))))
|
||||
out.append((kind, h8(m.group('v') if 'v' in rx.groupindex else m.group(0))))
|
||||
return out
|
||||
|
||||
@@ -38,7 +38,7 @@ FAILED=0
|
||||
|
||||
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
||||
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox}"
|
||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell}"
|
||||
|
||||
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
||||
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
||||
|
||||
Reference in New Issue
Block a user