Compare commits
24 Commits
9566826ce9
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| d609aa942a | |||
|
|
0fb2df11a6 | ||
| 44a1800eff | |||
|
|
dc1cfbf044 | ||
| 61776ce021 | |||
|
|
5c42b0e760 | ||
| 74ca5ac19b | |||
|
|
5347c11f69 | ||
|
|
5fa1e652ae | ||
|
|
6f19e23eaf | ||
| ef96051d6f | |||
|
|
a0dc6f8568 | ||
|
|
51e0b9d30b | ||
|
|
51777b0ad0 | ||
|
|
cbcb4c206b | ||
|
|
2c62e2834a | ||
| 3a9b7ecab7 | |||
|
|
cd0400c371 | ||
|
|
4e7ab667ed | ||
|
|
f10db19316 | ||
|
|
3503894a1e | ||
|
|
fbab5c4669 | ||
|
|
1b552c0882 | ||
|
|
8ebc18c3bc |
7
.github/CODEOWNERS
vendored
Normal file
7
.github/CODEOWNERS
vendored
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
# Changes to the guard allow-lists / exemptions need review by the account owner on GitHub, AND an
|
||||||
|
# approved_by (windy-hub | windy-mind) on every non-structural entry, which the guard enforces itself
|
||||||
|
# (scripts/compute_guard.py: load_allow). A lane never approves its own exemption (Hub 10-02).
|
||||||
|
/ci/compute-guard-allow.yml @sneakyfree
|
||||||
|
/ci/secret-guard-allow.yml @sneakyfree
|
||||||
|
/ci/ci-hygiene-allow.yml @sneakyfree
|
||||||
|
/scripts/compute_guard.py @sneakyfree
|
||||||
@@ -86,7 +86,7 @@ def test_warn_mode_never_turns_red(monkeypatch):
|
|||||||
def test_allow_file_is_line_scoped_exceptions_only():
|
def test_allow_file_is_line_scoped_exceptions_only():
|
||||||
"""Every exception is line-scoped (`matches`), so an allowed file can't hide a
|
"""Every exception is line-scoped (`matches`), so an allowed file can't hide a
|
||||||
NEW floating install or docker step. Today: windy-pro's if:false deploy job."""
|
NEW floating install or docker step. Today: windy-pro's if:false deploy job."""
|
||||||
allow = hy.cg.load_allow(hy.ALLOW_FILE)
|
allow = hy.cg.load_allow(hy.ALLOW_FILE, strict=False)
|
||||||
assert [(e["repo"], e["paths"]) for e in allow] == [("windy-pro", [".github/workflows/ci.yml"])]
|
assert [(e["repo"], e["paths"]) for e in allow] == [("windy-pro", [".github/workflows/ci.yml"])]
|
||||||
assert all(e.get("matches") for e in allow)
|
assert all(e.get("matches") for e in allow)
|
||||||
ok = " run: docker build -f account-server/Dockerfile -t windy-pro:${{ github.sha }} ."
|
ok = " run: docker build -f account-server/Dockerfile -t windy-pro:${{ github.sha }} ."
|
||||||
|
|||||||
@@ -77,6 +77,88 @@ def test_allow_list_needs_a_reason_per_entry(tmp_path):
|
|||||||
cg.load_allow(bad)
|
cg.load_allow(bad)
|
||||||
|
|
||||||
|
|
||||||
|
def _entry(**kw):
|
||||||
|
base = dict(repo="x", paths=["*"], reason="r", exemption="owner-approved", expires="2099-01-01",
|
||||||
|
approved_by="windy-hub")
|
||||||
|
base.update(kw)
|
||||||
|
lines = ["allow:", " - repo: x", " paths: ['*']", " reason: r"]
|
||||||
|
for k in ("exemption", "expires", "approved_by"):
|
||||||
|
if base.get(k) is not None:
|
||||||
|
lines.append(f" {k}: {base[k]}")
|
||||||
|
return "\n".join(lines) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_entries_need_a_named_exemption_and_an_expiry(tmp_path):
|
||||||
|
from datetime import date
|
||||||
|
f = tmp_path / "a.yml"
|
||||||
|
f.write_text(_entry(exemption=None))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f)
|
||||||
|
f.write_text(_entry(exemption="because-i-said-so"))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f)
|
||||||
|
f.write_text(_entry(expires=None))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f)
|
||||||
|
f.write_text(_entry(expires="someday"))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f)
|
||||||
|
f.write_text(_entry(expires="2026-12-01"))
|
||||||
|
assert len(cg.load_allow(f, today=date(2026, 10, 2))) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_expired_exemption_stops_excusing_and_is_reported(tmp_path):
|
||||||
|
from datetime import date
|
||||||
|
f = tmp_path / "a.yml"
|
||||||
|
f.write_text(_entry(expires="2026-10-01"))
|
||||||
|
assert cg.load_allow(f, today=date(2026, 10, 1)) # the expiry day is still valid
|
||||||
|
assert cg.load_allow(f, today=date(2026, 10, 2)) == [] # the next day it no longer excuses anything
|
||||||
|
assert cg.EXPIRED and cg.EXPIRED[0]["repo"] == "x" and cg.EXPIRED[0]["expires"] == "2026-10-01"
|
||||||
|
|
||||||
|
|
||||||
|
def test_shipped_allow_file_is_valid_today():
|
||||||
|
assert cg.load_allow() and not cg.EXPIRED # nothing in the repo's own file may already be expired
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text, kind", [
|
||||||
|
('u = "https://api.deepgram.com/v1/listen"', "voice-ai host"),
|
||||||
|
("fetch(`https://api.elevenlabs.io/v1/tts`)", "voice-ai host"),
|
||||||
|
('h = "api.cartesia.ai"', "voice-ai host"),
|
||||||
|
('h = "app.resemble.ai"', "voice-ai host"),
|
||||||
|
('h = "speech.googleapis.com"', "voice-ai host"),
|
||||||
|
('h = "transcribe.us-east-1.amazonaws.com"', "voice-ai host"),
|
||||||
|
('h = "polly.eu-west-1.amazonaws.com"', "voice-ai host"),
|
||||||
|
("DEEPGRAM_API_KEY=abc", "voice-ai key"),
|
||||||
|
("ELEVENLABS_API_KEY = x", "voice-ai key"),
|
||||||
|
('u = f"https://api.cloudflare.com/client/v4/accounts/{a}/ai/run/@cf/m"', "cloudflare workers ai"),
|
||||||
|
('ENGINE = "http://10.0.0.5:8791/v1"', "talk engine port"),
|
||||||
|
('ENGINE = "http://h:8788/ws"', "talk engine port"),
|
||||||
|
('x = "http://h:8794/health"', "talk engine port"),
|
||||||
|
])
|
||||||
|
def test_gatekeeper_rules_fire(text, kind):
|
||||||
|
assert kind in [k for k, _ in cg.scan_line("app/x.py", text)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_workers_ai_binding_only_in_wrangler_and_near_misses_are_quiet():
|
||||||
|
assert [k for k, _ in cg.scan_line("wrangler.toml", "[ai]")] == ["workers ai binding"]
|
||||||
|
assert [k for k, _ in cg.scan_line("apps/x/wrangler.jsonc", ' "ai": {')] == ["workers ai binding"]
|
||||||
|
assert cg.scan_line("other.toml", "[ai]") == []
|
||||||
|
assert cg.scan_line("app/x.py", "port = 87912") == []
|
||||||
|
assert cg.scan_line("app/x.py", "# talk engine was :8791 (removed)") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_rolling_out_kinds_warn_but_dont_block_and_hard_kinds_still_do(monkeypatch):
|
||||||
|
monkeypatch.setattr(cg, "MODE", "block")
|
||||||
|
monkeypatch.setattr(cg, "SOFT_KINDS", {"voice-ai host", "voice-ai key"})
|
||||||
|
soft = cg.Finding("a.py", 1, "voice-ai host", "api.deepgram.com")
|
||||||
|
hard = cg.Finding("a.py", 2, "provider host", "api.openai.com")
|
||||||
|
state, desc, _ = cg.status_for([soft], whole_tree=True)
|
||||||
|
assert state == "success" and "rolling out" in desc and len(desc) <= 140
|
||||||
|
assert cg.status_for([soft, hard], whole_tree=True)[0] == "failure"
|
||||||
|
monkeypatch.setattr(cg, "SOFT_KINDS", set())
|
||||||
|
assert cg.status_for([soft], whole_tree=True)[0] == "failure" # rollout over: it blocks
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize(
|
@pytest.mark.parametrize(
|
||||||
"repo, path, ok",
|
"repo, path, ok",
|
||||||
[
|
[
|
||||||
@@ -229,3 +311,62 @@ def test_block_mode_never_blocks_grant_owned(monkeypatch):
|
|||||||
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
||||||
lane = cg.Finding("a.py", 3, "provider host", "x")
|
lane = cg.Finding("a.py", 3, "provider host", "x")
|
||||||
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
||||||
|
|
||||||
|
|
||||||
|
def test_veron_ollama_warns_on_added_lines_and_never_blocks(monkeypatch):
|
||||||
|
hits = cg.scan_line("app/llm.py", 'OLLAMA = "http://192.168.1.73:11434/api/generate"')
|
||||||
|
assert [k for k, _ in hits] == ["veron ollama"]
|
||||||
|
assert cg.scan_line("app/llm.py", 'port = 114345') == [] # not the port
|
||||||
|
assert cg.scan_line("app/llm.py", "# was http://x:11434 (removed)") == [] # a comment is not a call
|
||||||
|
f = cg.Finding("app/llm.py", 7, "veron ollama", ":11434")
|
||||||
|
monkeypatch.setattr(cg, "MODE", "block")
|
||||||
|
state, desc, _ = cg.status_for([f], whole_tree=False)
|
||||||
|
assert state == "success" and desc.startswith("⚠ WARN: new Veron Ollama ref app/llm.py:7")
|
||||||
|
assert "Windy Mind" in desc and len(desc) <= 140
|
||||||
|
hard = cg.Finding("app/llm.py", 1, "provider host", "api.openai.com")
|
||||||
|
assert cg.status_for([f, hard], whole_tree=False)[0] == "failure" # a real violation still blocks
|
||||||
|
|
||||||
|
|
||||||
|
def test_ollama_in_added_pr_lines_only():
|
||||||
|
diff = ("+++ b/svc/client.py\n@@ -0,0 +1,2 @@\n+import httpx\n"
|
||||||
|
"+URL = 'http://veron:11434/api/chat'\n")
|
||||||
|
got = cg.parse_added("some-repo", diff, [])
|
||||||
|
assert [(f.kind, f.line) for f in got] == [("veron ollama", 2)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_structural_exemptions_need_an_independent_approver_and_a_90_day_cap(tmp_path):
|
||||||
|
from datetime import date
|
||||||
|
f = tmp_path / "a.yml"
|
||||||
|
f.write_text(_entry(approved_by=None))
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f, today=date(2026, 10, 2))
|
||||||
|
f.write_text(_entry(approved_by="windy-chat")) # a lane may not approve itself/another lane
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(f, today=date(2026, 10, 2))
|
||||||
|
f.write_text(_entry(expires="2026-12-31")) # exactly 90 days: fine
|
||||||
|
assert len(cg.load_allow(f, today=date(2026, 10, 2))) == 1
|
||||||
|
f.write_text(_entry(expires="2027-01-01")) # 91 days: does NOT apply, and is reported
|
||||||
|
assert cg.load_allow(f, today=date(2026, 10, 2)) == [] and cg.OVERCAP
|
||||||
|
f.write_text(_entry(exemption="compute-door", approved_by=None, expires="2027-10-02"))
|
||||||
|
assert len(cg.load_allow(f, today=date(2026, 10, 2))) == 1 # structural: yearly, no approver field
|
||||||
|
|
||||||
|
|
||||||
|
def test_shipped_allow_file_obeys_its_own_rules():
|
||||||
|
allow = cg.load_allow()
|
||||||
|
assert allow and not cg.EXPIRED and not cg.OVERCAP
|
||||||
|
for e in allow:
|
||||||
|
if e["exemption"] not in cg.STRUCTURAL:
|
||||||
|
assert e["approved_by"] in cg.APPROVERS
|
||||||
|
|
||||||
|
|
||||||
|
def test_findings_carry_kind_and_name_never_the_value_and_ports_skip_contracts():
|
||||||
|
for line, kind in [("ELEVENLABS_API_KEY=sk_live_SUPERSECRET123456789", "voice-ai key"),
|
||||||
|
('DEEPGRAM_API_KEY = "dg-VALUE-0123456789abcdef"', "voice-ai key")]:
|
||||||
|
hits = cg.scan_line("app/x.py", line)
|
||||||
|
assert [k for k, _ in hits] == [kind]
|
||||||
|
assert all("SUPERSECRET" not in m and "VALUE" not in m for _, m in hits)
|
||||||
|
assert cg.scan_line("engine/contracts/ops.mcp.v1.json", '"url": "http://h:8791/x"') == []
|
||||||
|
assert cg.scan_line("services/api/openapi/spec.json", '"url": "http://h:8791/x"') == []
|
||||||
|
assert [k for k, _ in cg.scan_line("deploy/docker-compose.yml", " - 8791:8791 # :8791")] == ["talk engine port"]
|
||||||
|
# :8099 is windy-pro's OLD translate-api / cloud-storage service, NOT the Talk engine (Hub 10-02): not flagged
|
||||||
|
assert cg.scan_line("deploy/docker-compose.yml", " - 8099:8099 # translate-api:8099") == []
|
||||||
|
|||||||
64
api/tests/test_lockbox_names.py
Normal file
64
api/tests/test_lockbox_names.py
Normal file
@@ -0,0 +1,64 @@
|
|||||||
|
"""lockbox-names: headings + labels + resolvable, NEVER a value or prose after a label."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
V1 = "Qm7xT2vLp9RkZw4HnB8dYc3S" # synthetic values
|
||||||
|
V2 = "other-fake-value-1234567890"
|
||||||
|
V3 = "dup-one-aaaaaaaaaaaaaaaa"
|
||||||
|
V4 = "dup-two-bbbbbbbbbbbbbbbb"
|
||||||
|
PROSE = "ZZPROSEZZ-not-for-printing"
|
||||||
|
|
||||||
|
|
||||||
|
def make(tmp_path):
|
||||||
|
r = tmp_path / "kit"
|
||||||
|
(r / "secrets" / "x").mkdir(parents=True)
|
||||||
|
(r / "ACCESS_LOCKBOX.md").write_text(
|
||||||
|
"# LOCKBOX\n\n## 🔷 AZURE signing (added 10-01)\n"
|
||||||
|
f"- **Tenant:** {PROSE} lives in the portal\n"
|
||||||
|
f"- **`AZURE_CLIENT_ID`**: `{V1}`\n"
|
||||||
|
f"- **Secret (AZURE_CLIENT_SECRET):** `{V2}`\n"
|
||||||
|
"## GOOGLE oauth\n"
|
||||||
|
f"GOOGLE_OAUTH_CLIENT_ID={V2}\n"
|
||||||
|
f"- **`DUP_KEY`**: `{V3}`\n- **`DUP_KEY`**: `{V4}`\n"
|
||||||
|
f"## stray\n**{V1}** is a heading-like bold that is secret shaped? no, just label\n")
|
||||||
|
(r / "secrets" / "x" / "a.env").write_text(f"FILE_KEY={V1}\n")
|
||||||
|
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=r, check=True)
|
||||||
|
return r
|
||||||
|
|
||||||
|
|
||||||
|
def run(r, *args):
|
||||||
|
e = {**os.environ, "LOCKBOX_REPO": str(r), "LOCKBOX_REF": "WORKTREE"}
|
||||||
|
p = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_names.py"), *args],
|
||||||
|
capture_output=True, text=True, env=e)
|
||||||
|
return p.returncode, p.stdout + p.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_lists_labels_and_resolvable_without_values_or_prose(tmp_path):
|
||||||
|
r = make(tmp_path)
|
||||||
|
rc, out = run(r, "AZURE|GOOGLE|FILE|DUP")
|
||||||
|
assert rc == 0
|
||||||
|
assert "AZURE signing (added 10-01) | AZURE_CLIENT_ID | md | yes" in out
|
||||||
|
assert "| GOOGLE_OAUTH_CLIENT_ID | env | yes" in out
|
||||||
|
assert "| FILE_KEY | file | yes" in out
|
||||||
|
assert "| DUP_KEY | md | dup" in out
|
||||||
|
# a prose label is listed but never resolvable, and nothing after the label leaks
|
||||||
|
assert "| Tenant: " not in out or "| Tenant" in out
|
||||||
|
assert "| label | no" in out
|
||||||
|
for secret in (V1, V2, V3, V4, PROSE, "lives in the portal"):
|
||||||
|
assert secret not in out
|
||||||
|
for i in range(0, len(secret) - 7):
|
||||||
|
assert secret[i:i + 8] not in out
|
||||||
|
|
||||||
|
|
||||||
|
def test_filter_and_bad_regex(tmp_path):
|
||||||
|
r = make(tmp_path)
|
||||||
|
rc, out = run(r, "NOSUCHTHING")
|
||||||
|
assert rc == 0 and "0 entries" in out
|
||||||
|
rc, out = run(r, "(")
|
||||||
|
assert rc == 2 and "bad regex" in out
|
||||||
70
api/tests/test_runner_guard.py
Normal file
70
api/tests/test_runner_guard.py
Normal file
@@ -0,0 +1,70 @@
|
|||||||
|
"""runner-guard: workflow shapes that hand a self-hosted runner to strangers."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_spec = importlib.util.spec_from_file_location("runner_guard", ROOT / "scripts" / "runner_guard.py")
|
||||||
|
rg = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["runner_guard"] = rg
|
||||||
|
_spec.loader.exec_module(rg)
|
||||||
|
|
||||||
|
|
||||||
|
def rules(text):
|
||||||
|
return [(r, ln) for _p, ln, r, _m in rg.lint_text("w.yml", text)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_pull_request_target_always_fails():
|
||||||
|
assert rules("on: pull_request_target\njobs:\n a:\n runs-on: ubuntu-latest\n steps: []\n")[0][0] == "R1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_fork_pr_on_self_hosted_fails_and_points_at_runs_on():
|
||||||
|
wf = "on:\n pull_request:\njobs:\n t:\n runs-on: [self-hosted, linux, x64]\n steps: []\n"
|
||||||
|
assert rules(wf) == [("R2", 5)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_repo_gate_or_environment_passes():
|
||||||
|
gated = ("on: [pull_request]\njobs:\n t:\n if: github.event.pull_request.head.repo.full_name == github.repository\n"
|
||||||
|
" runs-on: [self-hosted]\n steps: []\n")
|
||||||
|
env = "on: [pull_request]\njobs:\n t:\n environment: ci\n runs-on: self-hosted\n steps: []\n"
|
||||||
|
assert rules(gated) == [] and rules(env) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_outsider_events_on_self_hosted_fail_but_writer_events_pass():
|
||||||
|
wf = "on:\n issue_comment:\n workflow_run:\n workflows: [x]\njobs:\n t:\n runs-on: self-hosted\n steps: []\n"
|
||||||
|
assert sorted(r for r, _ in rules(wf)) == ["R3", "R3"]
|
||||||
|
ok = "on:\n push:\n tags: ['v*']\n workflow_dispatch:\n schedule:\n - cron: '0 3 * * *'\njobs:\n t:\n runs-on: self-hosted\n steps: []\n"
|
||||||
|
assert rules(ok) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_expression_runs_on_is_treated_as_self_hosted_and_hosted_runner_is_fine():
|
||||||
|
expr = "on: pull_request\njobs:\n t:\n runs-on: ${{ matrix.os }}\n steps: []\n"
|
||||||
|
hosted = "on: pull_request\njobs:\n t:\n runs-on: ubuntu-latest\n steps: []\n"
|
||||||
|
assert rules(expr) == [("R2", 4)] and rules(hosted) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_broken_yaml_is_a_finding_and_non_workflows_are_ignored():
|
||||||
|
assert rules("on: [push\njobs: {")[0][0] == "R0"
|
||||||
|
assert rules("name: just a file\n") == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_pr_mode_posts_each_status_once(monkeypatch, tmp_path):
|
||||||
|
calls = []
|
||||||
|
monkeypatch.setattr(rg, "STATE", str(tmp_path / "s.json"))
|
||||||
|
monkeypatch.setattr(rg, "public_repos", lambda owners: [("o/r", "main")])
|
||||||
|
monkeypatch.setattr(rg, "file_at", lambda *a: "on: push\\njobs:\\n t:\\n runs-on: self-hosted\\n steps: []\\n")
|
||||||
|
|
||||||
|
def fake_gh(*args, check=True):
|
||||||
|
if args[0].startswith("repos/o/r/pulls?"):
|
||||||
|
return "7 abc123 o/r\\n"
|
||||||
|
if args[0].endswith("/files?per_page=100"):
|
||||||
|
return ".github/workflows/ci.yml\\n"
|
||||||
|
calls.append(args[0])
|
||||||
|
return ""
|
||||||
|
monkeypatch.setattr(rg, "gh", fake_gh)
|
||||||
|
rg.cmd_pr(["o"], post=True)
|
||||||
|
rg.cmd_pr(["o"], post=True)
|
||||||
|
assert calls == ["repos/o/r/statuses/abc123"]
|
||||||
@@ -146,3 +146,11 @@ def test_warn_kinds_do_not_block(monkeypatch):
|
|||||||
assert sg.status_for([f], True)[0] == "success"
|
assert sg.status_for([f], True)[0] == "success"
|
||||||
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
||||||
assert sg.status_for([f], True)[0] == "failure"
|
assert sg.status_for([f], True)[0] == "failure"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pypi_token_shape_hash_only():
|
||||||
|
tok = "pypi-AgE" + "Ab1_-" * 20 # synthetic
|
||||||
|
got = ss.find(f"password = {tok}")
|
||||||
|
assert [k for k, _ in got] == ["pypi token"] and got[0][1] == ss.h8(tok)
|
||||||
|
assert tok not in repr(got)
|
||||||
|
assert ss.find("pypi-AgE-too-short") == []
|
||||||
|
|||||||
@@ -2,11 +2,16 @@
|
|||||||
# Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23). Every entry
|
# Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23). Every entry
|
||||||
# here is an exception to that rule and MUST say why. Paths are fnmatch globs
|
# here is an exception to that rule and MUST say why. Paths are fnmatch globs
|
||||||
# relative to the repo root. Owner of this file: Windy Git lane (13); changes
|
# relative to the repo root. Owner of this file: Windy Git lane (13); changes
|
||||||
# go through the orchestrator. Source of the first entries: COMPUTE_BYPASS_AUDIT.md.
|
# go through the orchestrator. EVERY entry needs `exemption` (local-user-hardware | owner-approved |
|
||||||
|
# compute-door | guard-self) and `expires` (YYYY-MM-DD): nothing gets permanent amnesty (Mind 10-02);
|
||||||
|
# non-structural exemptions also need approved_by (windy-hub | windy-mind; a lane never approves its own)
|
||||||
|
# and expire within 90 days; an expired entry stops excusing code on that date and shows in the guard report. Source of the first entries: COMPUTE_BYPASS_AUDIT.md.
|
||||||
allow:
|
allow:
|
||||||
- repo: windy-mind
|
- repo: windy-mind
|
||||||
paths: ["*"]
|
paths: ["*"]
|
||||||
reason: "Windy Mind IS the door: provider clients belong here by definition."
|
reason: "Windy Mind IS the door: provider clients belong here by definition."
|
||||||
|
exemption: compute-door
|
||||||
|
expires: 2027-10-02
|
||||||
|
|
||||||
- repo: windy-agent
|
- repo: windy-agent
|
||||||
paths: ["*"]
|
paths: ["*"]
|
||||||
@@ -14,14 +19,26 @@ allow:
|
|||||||
User BYOK: self-hosted agents call providers on the USER's own keys.
|
User BYOK: self-hosted agents call providers on the USER's own keys.
|
||||||
Mind stays opt-in there, or every self-hosted user's inference lands on
|
Mind stays opt-in there, or every self-hosted user's inference lands on
|
||||||
Grant's bill (no-cloud-cost-liability rule; audit #7).
|
Grant's bill (no-cloud-cost-liability rule; audit #7).
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-code
|
- repo: windy-code
|
||||||
paths: ["extensions/windy-ai/*"]
|
paths: ["extensions/windy-ai/*"]
|
||||||
reason: "User BYOK AI extension: the user's own provider keys; Mind is one opt-in provider (audit #8)."
|
reason: "User BYOK AI extension: the user's own provider keys; Mind is one opt-in provider (audit #8)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-connect
|
- repo: windy-connect
|
||||||
paths: ["*writers/*"]
|
paths: ["*writers/*"]
|
||||||
reason: "Writes client configs that NAME the user's own provider env vars; makes no provider calls (audit #11)."
|
reason: "Writes client configs that NAME the user's own provider env vars; makes no provider calls (audit #11)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-pro
|
- repo: windy-pro
|
||||||
paths: ["src/client/desktop/*"]
|
paths: ["src/client/desktop/*"]
|
||||||
@@ -30,10 +47,18 @@ allow:
|
|||||||
enters (renderer localStorage -> electron-store; env var only for dev), and
|
enters (renderer localStorage -> electron-store; env var only for dev), and
|
||||||
the CSP line allows exactly those user-keyed hosts (audit #10). The
|
the CSP line allows exactly those user-keyed hosts (audit #10). The
|
||||||
account-server is NOT covered: server-side calls go through Mind.
|
account-server is NOT covered: server-side calls go through Mind.
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-pro
|
- repo: windy-pro
|
||||||
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
||||||
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-pro
|
- repo: windy-pro
|
||||||
paths: ["src/client/web/src/pages/panels/MindKeychain.jsx"]
|
paths: ["src/client/web/src/pages/panels/MindKeychain.jsx"]
|
||||||
@@ -41,7 +66,65 @@ allow:
|
|||||||
# /api/v1/chat, i.e. inference) still flags. Orchestrator-approved 09-23.
|
# /api/v1/chat, i.e. inference) still flags. Orchestrator-approved 09-23.
|
||||||
matches: ['openrouter\.ai/auth\?', 'openrouter\.ai/api/v1/auth/keys']
|
matches: ['openrouter\.ai/auth\?', 'openrouter\.ai/api/v1/auth/keys']
|
||||||
reason: "BYOK key acquisition via OpenRouter OAuth PKCE; no inference; successor of MindPanel allow (ADR-064)."
|
reason: "BYOK key acquisition via OpenRouter OAuth PKCE; no inference; successor of MindPanel allow (ADR-064)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
- repo: windy-git
|
- repo: windy-git
|
||||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||||
reason: "The guard's own pattern list and this file."
|
reason: "The guard's own pattern list and this file."
|
||||||
|
exemption: guard-self
|
||||||
|
expires: 2027-10-02
|
||||||
|
|
||||||
|
- repo: windy-mind
|
||||||
|
paths: ["*"]
|
||||||
|
matches: [':11434']
|
||||||
|
reason: "Windy Mind IS the compute door (endpoint + key); it may call Ollama. Only the Ollama port is allowed here, any provider host/SDK in Mind still flags."
|
||||||
|
exemption: compute-door
|
||||||
|
expires: 2027-10-02
|
||||||
|
|
||||||
|
- repo: windy-pro
|
||||||
|
paths: [".env.example"]
|
||||||
|
matches: ['DEEPGRAM_API_KEY']
|
||||||
|
reason: "Template line (name only, no value) for the existing user-own-key Deepgram feature in Word's Settings. Mind's migration removes the feature and then this line (Hub classification 10-02)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
|
- repo: windy-registry
|
||||||
|
paths: ["tools/r2-provision.sh"]
|
||||||
|
matches: ['http://localhost:8788']
|
||||||
|
reason: "Dev origin in an R2 bucket CORS rule, not a call to the Talk engine (Hub 10-02)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
|
- repo: windytalk
|
||||||
|
paths: ["engine/*", "scripts/stress/*", "scripts/veron/*"]
|
||||||
|
matches: [':(8791|8788|8794)']
|
||||||
|
reason: "Talk's own voice engines (server, systemd unit, stress scripts); to be placed behind Mind per Mind's audit plan. NOT compute-door: a real bypass being fixed (Hub 10-02)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
|
- repo: windytalk
|
||||||
|
paths: ["scripts/run-client.sh"]
|
||||||
|
matches: [':(8791|8788|8794)']
|
||||||
|
reason: "Dev client launcher: 127.0.0.1:8788 is an ssh -L tunnel to the dev engine ON VERON (not the user's machine), so engine-side; dev-only, not shipped. To be placed behind Mind per Mind's audit plan (Windy Talk, Hub 10-02)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|
||||||
|
- repo: windytalk
|
||||||
|
paths: ["apps/desktop/renderer/index.html"]
|
||||||
|
matches: [':(8791|8788|8794)']
|
||||||
|
reason: "Display-only fallback label in the settings panel (cfg.engineUrl || default); connects to nothing. The shipped client defaults to the public engine on Veron, never local inference (Windy Talk, Hub 10-02)."
|
||||||
|
exemption: owner-approved
|
||||||
|
approved_by: windy-hub
|
||||||
|
approved_on: 2026-10-02
|
||||||
|
expires: 2026-12-31
|
||||||
|
|||||||
@@ -57,3 +57,6 @@ allow:
|
|||||||
- repo: windytalk
|
- repo: windytalk
|
||||||
hashes: ["c4189d79"]
|
hashes: ["c4189d79"]
|
||||||
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
||||||
|
- repo: windy-agent
|
||||||
|
hashes: ["84e0c0ea"]
|
||||||
|
reason: "tests/test_log_redaction.py:56 Z.ai redaction fixture REPLACED by windy-agent #412 with a synthetic value; hash checked against the lockbox 10-01: not present."
|
||||||
|
|||||||
14
deploy/systemd/windygit-state-backup.service
Normal file
14
deploy/systemd/windygit-state-backup.service
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2)
|
||||||
|
After=network-online.target docker.service
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
WorkingDirectory=/srv/windygit/src
|
||||||
|
# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass
|
||||||
|
# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD).
|
||||||
|
EnvironmentFile=/srv/windygit/src/.env
|
||||||
|
ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||||
|
Nice=10
|
||||||
|
IOSchedulingClass=idle
|
||||||
|
TimeoutStartSec=3h
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
[Service]
|
||||||
|
ExecStart=
|
||||||
|
ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||||
11
deploy/systemd/windygit-state-backup.timer
Normal file
11
deploy/systemd/windygit-state-backup.timer
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Nightly Windy Git state backup
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
# 03:07 local, clear of the git-bundle backup at 04:17.
|
||||||
|
OnCalendar=*-*-* 03:07:00
|
||||||
|
Persistent=true
|
||||||
|
RandomizedDelaySec=300
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
28
deploy/windy0/nightly-runner-guard-sweep.sh
Executable file
28
deploy/windy0/nightly-runner-guard-sweep.sh
Executable file
@@ -0,0 +1,28 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Nightly (Boss 10-01): runner-guard over the default branch of EVERY public repo in Grant's
|
||||||
|
# 5 GitHub accounts (runs on Veron: windy-git scripts/runner_guard.py report). Writes
|
||||||
|
# ~/windy-orchestra/RUNNER_GUARD.md (repo, file:line, rule; no file content), appends ONE
|
||||||
|
# BOARD line per NEW hit vs the last run, and prints "runner-guard sweep: N hit(s)" last, so
|
||||||
|
# the windy-job heartbeat (--expect "runner-guard sweep: 0 hit") goes red while any hit exists.
|
||||||
|
set -euo pipefail
|
||||||
|
page=~/windy-orchestra/RUNNER_GUARD.md
|
||||||
|
state=~/.local/state/runner-guard-sweep.txt
|
||||||
|
mkdir -p "$(dirname "$state")"
|
||||||
|
out=$(timeout 900 ssh -o BatchMode=yes -o ConnectTimeout=15 ts-veron \
|
||||||
|
'cd /srv/windygit/src && timeout 850 python3 scripts/runner_guard.py report' || true)
|
||||||
|
summary=$(grep '^# runner-guard sweep:' <<<"$out" || echo "# runner-guard sweep: ERROR (no summary)")
|
||||||
|
hits=$(grep -v '^#' <<<"$out" | grep . || true)
|
||||||
|
{
|
||||||
|
echo "# Runner guard: stranger-code paths to self-hosted runners ($(date -u '+%Y-%m-%d %H:%MZ'))"
|
||||||
|
echo "_Nightly; windy-git scripts/runner_guard.py. R1 pull_request_target · R2 fork PR on self-hosted without a same-repo/environment gate · R3 outsider events (issue_comment, workflow_run, ...) on self-hosted · R0 unparseable._"
|
||||||
|
echo; echo "${summary#\# }"; echo
|
||||||
|
echo "| repo | file:line | rule | fix |"; echo "|---|---|---|---|"
|
||||||
|
while IFS=$'\t' read -r repo loc rule msg; do [[ -n $repo ]] && echo "| $repo | $loc | $rule | $msg |"; done <<<"$hits"
|
||||||
|
} > "$page"
|
||||||
|
new=$(comm -13 <(sort -u "$state" 2>/dev/null || true) <(cut -f1-3 <<<"$hits" | sort -u))
|
||||||
|
cut -f1-3 <<<"$hits" | sort -u > "$state"
|
||||||
|
if [[ -n "$new" ]]; then
|
||||||
|
n=$(grep -c . <<<"$new")
|
||||||
|
echo "$(date -u +%Y-%m-%dT%H:%MZ) Windy Git: 🚨 runner-guard: $n NEW stranger-code path(s) to a self-hosted runner in public repos; see ~/windy-orchestra/RUNNER_GUARD.md" >> ~/windy-orchestra/BOARD.md
|
||||||
|
fi
|
||||||
|
echo "${summary#\# }"
|
||||||
7
deploy/windy0/windy-runner-guard-sweep.service
Normal file
7
deploy/windy0/windy-runner-guard-sweep.service
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Nightly runner-guard sweep of every PUBLIC repo's workflows (Windy Git lane)
|
||||||
|
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/local/bin/windy-job windy-runner-guard-sweep 26h --expect "runner-guard sweep: 0 hit" --owner 13 -- %h/bin/nightly-runner-guard-sweep.sh
|
||||||
|
TimeoutStartSec=1200
|
||||||
9
deploy/windy0/windy-runner-guard-sweep.timer
Normal file
9
deploy/windy0/windy-runner-guard-sweep.timer
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
[Unit]
|
||||||
|
Description=Nightly runner-guard sweep
|
||||||
|
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*-*-* 09:40:00 UTC
|
||||||
|
Persistent=true
|
||||||
|
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
@@ -32,7 +32,7 @@ services:
|
|||||||
|
|
||||||
gitea:
|
gitea:
|
||||||
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
||||||
image: docker.io/gitea/gitea:1.24.6
|
image: docker.io/gitea/gitea:1.24.7
|
||||||
environment:
|
environment:
|
||||||
GITEA__database__DB_TYPE: postgres
|
GITEA__database__DB_TYPE: postgres
|
||||||
GITEA__database__HOST: db:5432
|
GITEA__database__HOST: db:5432
|
||||||
|
|||||||
31
docs/CUTOVER-PRIMARY.md
Normal file
31
docs/CUTOVER-PRIMARY.md
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
# Checklist: making Windy Git the PRIMARY home of one repo (after launch)
|
||||||
|
|
||||||
|
Status 2026-10-01: DRAFT, nothing flipped. GitHub stays the source of truth until Grant says otherwise.
|
||||||
|
First candidate: `windy-git` itself (public, low blast radius). GitHub becomes the free off-site push-mirror.
|
||||||
|
|
||||||
|
## Preconditions (all must be true)
|
||||||
|
- [x] Encrypted state backup (Postgres + Gitea config + repos) nightly, restore drill passed cross-host (10-01).
|
||||||
|
- [x] Gitea on a patched release (1.24.7); upgrade path = snapshot first (docs/RESTORE-DRILL.md).
|
||||||
|
- [ ] Heartbeat `windygit-state-backup` in heartbeats-veron expected list (asked Cloud/Super Admin 10-01).
|
||||||
|
- [ ] A missed/failed backup PAGES (heartbeat 26h), tested once by skipping a night in a drill.
|
||||||
|
- [ ] Boss/Grant capacity call on Veron (dedicated non-SMR volume for DB + git storage; root disk < 80%).
|
||||||
|
- [ ] Post-launch freeze lifted (Hub). No cutover during store review or a launch window.
|
||||||
|
|
||||||
|
## Cutover for ONE repo (reversible at every step)
|
||||||
|
1. Announce on BOARD; tell every consuming lane (no schema drift rule). Pause the sync for that repo only:
|
||||||
|
remove it from `REPOS` in `scripts/sync_from_github.sh` FIRST (the sync force-overwrites Windy Git).
|
||||||
|
2. Verify Windy Git main == GitHub main (sha equal), all branches/tags present, LFS (if any) in R2.
|
||||||
|
3. Add a PUSH-mirror on the Windy Git repo -> GitHub (deploy key or scoped token, write on that repo only,
|
||||||
|
interval 8h + on-commit), so GitHub keeps a current copy. Test with a throwaway branch.
|
||||||
|
4. Flip the lanes' remote: `origin` = Windy Git (SSH/HTTPS via Windy SSO token), `github` = secondary.
|
||||||
|
Lanes push to Windy Git only; the mirror carries it to GitHub.
|
||||||
|
5. CI keeps running here (no change); remove the `pr_status_bridge` mirror-PR duplication for that repo
|
||||||
|
(PRs are now native here; the bridge's GitHub status posting for it can stay for any open GitHub PRs).
|
||||||
|
6. Watch 3 days: mirror lag, backup includes the new writes, no push rejected, no lane still pushing to GitHub.
|
||||||
|
7. Rollback at any time: re-add the repo to `REPOS` (sync resumes GitHub->Windy Git, GitHub is intact via the
|
||||||
|
push-mirror) and point lanes' remote back. Nothing is destroyed in either direction.
|
||||||
|
|
||||||
|
## NOT in scope for a first cutover
|
||||||
|
Deploy workflows (stay disabled; deploys remain manual until a separate runner + scoped deploy keys exist),
|
||||||
|
credential repos (soul/anima/kit-army-config), windy-pro (importer refuses by name), opening the forge to
|
||||||
|
other members (Grant 09-23: registration closed; Hub 10-01: jit false, 4 conditions before any change).
|
||||||
33
docs/RESTORE-DRILL.md
Normal file
33
docs/RESTORE-DRILL.md
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
# Restoring Windy Git from the encrypted state backup
|
||||||
|
|
||||||
|
What is backed up (`scripts/backup_state.sh`, restic repo `s3:…/windy-git-backups/restic`, tag `windygit-state`):
|
||||||
|
both Postgres databases (`gitea`, `windygit`, custom-format dumps + globals), the whole Gitea data root
|
||||||
|
(`/srv/windygit/git`: config, jwt, attachments, avatars, templates AND the bare repositories),
|
||||||
|
`/srv/windygit/src/.env`, `deploy/runner/.env`, `/etc/cloudflared`, the windygit systemd drop-ins.
|
||||||
|
NOT in it: the restic password itself (lockbox `RESTIC_WINDYGIT_PASSWORD`) and the R2 access key
|
||||||
|
(scoped token `windy-git-r2-scoped`, lockbox). Never print either: use `lockbox-get KEY FILE`.
|
||||||
|
|
||||||
|
## Drill (any machine with restic + docker; proven on Veron 2026-10-01, counts identical)
|
||||||
|
export RESTIC_PASSWORD_FILE=<0600 file from lockbox-get RESTIC_WINDYGIT_PASSWORD>
|
||||||
|
export AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… # from lockbox-get, into env, not echoed
|
||||||
|
export RESTIC_REPOSITORY=s3:https://<R2 account>.r2.cloudflarestorage.com/windy-git-backups/restic
|
||||||
|
restic snapshots --tag windygit-state
|
||||||
|
restic restore latest --tag windygit-state --target /var/tmp/wg-drill
|
||||||
|
docker run -d --name wg-drill-pg -e POSTGRES_PASSWORD=<random> -e POSTGRES_USER=drill postgres:16-alpine
|
||||||
|
for db in gitea windygit; do
|
||||||
|
docker exec wg-drill-pg psql -U drill -d postgres -c "create database $db"
|
||||||
|
docker exec -i wg-drill-pg pg_restore -U drill -d $db --no-owner --no-privileges \
|
||||||
|
< /var/tmp/wg-drill/var/backups/windygit-state/$db.dump
|
||||||
|
done
|
||||||
|
# compare row counts with live (or with the last known): repository, issue, pull_request, "user",
|
||||||
|
# external_login_user, access_token, action_run, action_run_job
|
||||||
|
docker rm -f wg-drill-pg; rm -rf /var/tmp/wg-drill
|
||||||
|
|
||||||
|
## Real disaster (Veron lost)
|
||||||
|
1. New Linux host with Docker, a Cloudflare tunnel connector, the repo (`git clone` from GitHub: windy-git).
|
||||||
|
2. `restic restore latest --tag windygit-state --target /` (puts /srv/windygit/git, the .env files, /etc/cloudflared back).
|
||||||
|
3. `docker compose -p windy-git up -d db`, then pg_restore both dumps into it (as above, into the real db names/owner from `.env`).
|
||||||
|
4. `docker compose -p windy-git up -d` + `deploy/runner` runners; re-register runners if the token changed.
|
||||||
|
5. Verify: `/api/healthz`, Windy SSO login, `git ls-remote`, one CI run. GitHub is still the source of truth for code,
|
||||||
|
so repo content can also be re-synced from there; the database is what only this backup holds.
|
||||||
|
Retention: 14 daily / 8 weekly / 6 monthly (prune on Sundays). Integrity: every run does `restic check --read-data-subset=2%`.
|
||||||
@@ -22,7 +22,16 @@ GIT_ROOT="${GIT_DATA_ROOT:-/srv/windygit/git}"
|
|||||||
umask 077
|
umask 077
|
||||||
mkdir -p "$STAGE"; chmod 700 "$STAGE"; rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
mkdir -p "$STAGE"; chmod 700 "$STAGE"; rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||||
|
|
||||||
restic cat config >/dev/null 2>&1 || { log "initialising restic repo"; restic init >/dev/null; }
|
# systemd gives units no $HOME, and restic wants a cache dir: pin one.
|
||||||
|
export RESTIC_CACHE_DIR="${RESTIC_CACHE_DIR:-/var/cache/windygit-restic}"; mkdir -p "$RESTIC_CACHE_DIR"
|
||||||
|
if ! err=$(restic cat config 2>&1 >/dev/null); then
|
||||||
|
# only a MISSING repo may be initialised; any other error (auth, network, wrong password) must stop here
|
||||||
|
if grep -qiE "does not exist|is there a repository|unable to open config file" <<<"$err"; then
|
||||||
|
log "initialising restic repo"; restic init >/dev/null
|
||||||
|
else
|
||||||
|
log "FATAL: restic cannot open the repository: $(head -c 300 <<<"$err" | tr '\n' ' ')"; exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
PGU=$(timeout 30 docker exec "$DB" printenv POSTGRES_USER)
|
PGU=$(timeout 30 docker exec "$DB" printenv POSTGRES_USER)
|
||||||
[[ -n "$PGU" ]] || { log "FATAL: no POSTGRES_USER in $DB"; exit 1; }
|
[[ -n "$PGU" ]] || { log "FATAL: no POSTGRES_USER in $DB"; exit 1; }
|
||||||
|
|||||||
@@ -204,7 +204,7 @@ def _check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
|||||||
bare = cg.WORK / f"{repo}.git"
|
bare = cg.WORK / f"{repo}.git"
|
||||||
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
||||||
return None
|
return None
|
||||||
allow = cg.load_allow(ALLOW_FILE)
|
allow = cg.load_allow(ALLOW_FILE, strict=False)
|
||||||
rules = hashlib.sha256((PREFILTER + INCLUDE.pattern + EXACT_PY.pattern + EXACT_NPM.pattern).encode()).hexdigest()[:8]
|
rules = hashlib.sha256((PREFILTER + INCLUDE.pattern + EXACT_PY.pattern + EXACT_NPM.pattern).encode()).hexdigest()[:8]
|
||||||
fp = cg._fingerprint(allow) + ":" + rules # hashlib, not hash(): hash() is per-process random
|
fp = cg._fingerprint(allow) + ":" + rules # hashlib, not hash(): hash() is per-process random
|
||||||
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
||||||
@@ -232,7 +232,7 @@ def status_for(findings, whole_tree: bool, grant=()):
|
|||||||
|
|
||||||
|
|
||||||
def report(repos: list[str]) -> int:
|
def report(repos: list[str]) -> int:
|
||||||
allow = cg.load_allow(ALLOW_FILE)
|
allow = cg.load_allow(ALLOW_FILE, strict=False)
|
||||||
total = 0
|
total = 0
|
||||||
for repo in repos:
|
for repo in repos:
|
||||||
bare = cg.WORK / f"{repo}.git"
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ import re
|
|||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
|
from datetime import date, timedelta
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import yaml
|
import yaml
|
||||||
@@ -49,6 +50,17 @@ HOSTS = [
|
|||||||
"api.cohere.com", "api.fireworks.ai", "api.replicate.com",
|
"api.cohere.com", "api.fireworks.ai", "api.replicate.com",
|
||||||
"api-inference.huggingface.co",
|
"api-inference.huggingface.co",
|
||||||
]
|
]
|
||||||
|
# Mind's 10-02 gatekeeper list (speech / voice / avatar / vision / cloud ML): own kinds, so a rollout
|
||||||
|
# can be WARN-first (COMPUTE_GUARD_WARN_KINDS) without softening the original provider rules.
|
||||||
|
VOICE_HOSTS = [
|
||||||
|
"api.deepgram.com", "api.elevenlabs.io", "api.cartesia.ai", "api.play.ht", "api.playht.com",
|
||||||
|
"app.resemble.ai", "f.cluster.resemble.ai", "api.heygen.com",
|
||||||
|
"vision.googleapis.com", "speech.googleapis.com", "texttospeech.googleapis.com",
|
||||||
|
]
|
||||||
|
VOICE_KEYS = [
|
||||||
|
"DEEPGRAM_API_KEY", "ELEVENLABS_API_KEY", "ELEVEN_API_KEY", "CARTESIA_API_KEY", "PLAYHT_API_KEY",
|
||||||
|
"PLAY_HT_API_KEY", "PLAYHT_USER_ID", "RESEMBLE_API_KEY", "HEYGEN_API_KEY",
|
||||||
|
]
|
||||||
KEYS = [
|
KEYS = [
|
||||||
"ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN",
|
"ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN",
|
||||||
"OPENAI_API_KEY", "GROQ_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY",
|
"OPENAI_API_KEY", "GROQ_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY",
|
||||||
@@ -61,8 +73,23 @@ PY_SDKS = r"anthropic|openai|groq|mistralai|cohere|google\.generativeai|google\.
|
|||||||
JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/genai|@mistralai/mistralai"
|
JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/genai|@mistralai/mistralai"
|
||||||
r"|cohere-ai|together-ai|@ai-sdk/(?:anthropic|openai|groq|google|mistral)")
|
r"|cohere-ai|together-ai|@ai-sdk/(?:anthropic|openai|groq|google|mistral)")
|
||||||
|
|
||||||
|
# The engine-port rule is about CODE/CONFIG that calls the engine, not API contracts, schemas or specs.
|
||||||
|
PORT_SKIP = re.compile(r"(^|/)(contracts?|schemas?|specs?|openapi)/|\.json$", re.I)
|
||||||
|
WRANGLER = re.compile(r"(^|/)wrangler\.(toml|jsonc?)$")
|
||||||
|
WRANGLER_AI = re.compile(r'^\s*\[ai\]\s*$|^\s*"ai"\s*:\s*\{')
|
||||||
|
|
||||||
RULES: list[tuple[str, re.Pattern]] = [
|
RULES: list[tuple[str, re.Pattern]] = [
|
||||||
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||||
|
# Grant via Boss 10-01: compute = Windy Mind. A NEW reference to an Ollama port (Veron's :11434) is a
|
||||||
|
# direct call around Mind's metering/caps. WARN-only, never red, and only for lines a PR ADDS.
|
||||||
|
("veron ollama", re.compile(r"(?::|%3[aA])11434(?![0-9])")),
|
||||||
|
("voice-ai host", re.compile("|".join(re.escape(h) for h in VOICE_HOSTS))),
|
||||||
|
("voice-ai key", re.compile(r"\b(?:" + "|".join(VOICE_KEYS) + r")\b")),
|
||||||
|
("voice-ai host", re.compile(r"(?:transcribe|polly)\.[a-z0-9-]+\.amazonaws\.com")),
|
||||||
|
("provider host", re.compile(r"(?:bedrock-runtime|bedrock)\.[a-z0-9-]+\.amazonaws\.com")),
|
||||||
|
("cloudflare workers ai", re.compile(r"api\.cloudflare\.com/client/v4/accounts/[^\s'\"/]+/ai/")),
|
||||||
|
("talk engine port", re.compile(r"(?::|%3[aA])(?:8791|8788|8794)(?![0-9])")),
|
||||||
|
("workers ai binding", WRANGLER_AI),
|
||||||
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||||
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||||
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||||
@@ -70,6 +97,12 @@ RULES: list[tuple[str, re.Pattern]] = [
|
|||||||
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||||
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||||
]
|
]
|
||||||
|
# Kinds that never block (even in MODE=block) and are only judged on ADDED lines, never the baseline tree.
|
||||||
|
WARN_ONLY_KINDS = {"veron ollama"}
|
||||||
|
# Rolled out WARN-first: these kinds still show (tree + PRs) but never block, until the env var
|
||||||
|
# (a systemd drop-in on the sync, like SECRET_GUARD_WARN_KINDS) is removed.
|
||||||
|
SOFT_KINDS = {k for k in os.environ.get("COMPUTE_GUARD_WARN_KINDS", "").split(",") if k}
|
||||||
|
OLLAMA_MSG = "compute = Windy Mind (endpoint + key); do not call Veron's Ollama directly"
|
||||||
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||||
|
|
||||||
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||||
@@ -89,13 +122,47 @@ class Finding:
|
|||||||
match: str
|
match: str
|
||||||
|
|
||||||
|
|
||||||
def load_allow(path: Path = ALLOW_FILE) -> list[dict]:
|
EXEMPTIONS = {"local-user-hardware", "owner-approved", "compute-door", "guard-self"}
|
||||||
|
STRUCTURAL = {"compute-door", "guard-self"} # the door itself and the guard's own files: yearly review
|
||||||
|
APPROVERS = {"windy-hub", "windy-mind"} # a lane never approves its own exemption (Hub 10-02)
|
||||||
|
MAX_DAYS = 90 # every other exemption: 90 days max, then re-approve
|
||||||
|
EXPIRED: list[dict] = [] # entries dropped as expired on the last load_allow (reported, never silent)
|
||||||
|
OVERCAP: list[dict] = [] # entries dropped because their expiry is further out than MAX_DAYS
|
||||||
|
|
||||||
|
|
||||||
|
def load_allow(path: Path = ALLOW_FILE, today: date | None = None, strict: bool = True) -> list[dict]:
|
||||||
|
"""Active entries only. Every entry needs repo, paths, a reason, a NAMED exemption and an expiry
|
||||||
|
date (Mind 10-02: nothing gets permanent amnesty). An expired entry stops excusing code at once.
|
||||||
|
`strict=False` is for OTHER guards reusing this loader (ci-hygiene) with their own file format."""
|
||||||
|
today = today or date.today()
|
||||||
data = yaml.safe_load(path.read_text()) or {}
|
data = yaml.safe_load(path.read_text()) or {}
|
||||||
entries = data.get("allow") or []
|
entries = data.get("allow") or []
|
||||||
for e in entries: # a reason per entry is the whole point of the file
|
active = []
|
||||||
|
EXPIRED.clear()
|
||||||
|
OVERCAP.clear()
|
||||||
|
for e in entries:
|
||||||
if not (e.get("repo") and e.get("paths") and str(e.get("reason", "")).strip()):
|
if not (e.get("repo") and e.get("paths") and str(e.get("reason", "")).strip()):
|
||||||
raise ValueError(f"allow entry needs repo, paths and a reason: {e}")
|
raise ValueError(f"allow entry needs repo, paths and a reason: {e}")
|
||||||
return entries
|
if not strict:
|
||||||
|
active.append(e)
|
||||||
|
continue
|
||||||
|
if e.get("exemption") not in EXEMPTIONS:
|
||||||
|
raise ValueError(f"allow entry needs exemption in {sorted(EXEMPTIONS)}: {e.get('repo')} {e.get('paths')}")
|
||||||
|
try:
|
||||||
|
exp = e["expires"] if isinstance(e.get("expires"), date) else date.fromisoformat(str(e.get("expires")))
|
||||||
|
except ValueError as err:
|
||||||
|
raise ValueError(f"allow entry needs expires: YYYY-MM-DD: {e.get('repo')} {e.get('paths')}") from err
|
||||||
|
if e["exemption"] not in STRUCTURAL:
|
||||||
|
if e.get("approved_by") not in APPROVERS:
|
||||||
|
raise ValueError(f"allow entry needs approved_by in {sorted(APPROVERS)}: {e.get('repo')} {e.get('paths')}")
|
||||||
|
if exp > today + timedelta(days=MAX_DAYS):
|
||||||
|
OVERCAP.append({**e, "expires": exp.isoformat()}) # a longer amnesty simply does not apply
|
||||||
|
continue
|
||||||
|
if exp < today:
|
||||||
|
EXPIRED.append({**e, "expires": exp.isoformat()})
|
||||||
|
else:
|
||||||
|
active.append(e)
|
||||||
|
return active
|
||||||
|
|
||||||
|
|
||||||
def allowed(repo: str, path: str, allow: list[dict], text: str | None = None) -> bool:
|
def allowed(repo: str, path: str, allow: list[dict], text: str | None = None) -> bool:
|
||||||
@@ -126,6 +193,10 @@ def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
|||||||
for kind, rx in RULES:
|
for kind, rx in RULES:
|
||||||
if kind == "provider SDK dep" and not DEP_FILES.search(path):
|
if kind == "provider SDK dep" and not DEP_FILES.search(path):
|
||||||
continue
|
continue
|
||||||
|
if kind == "workers ai binding" and not WRANGLER.search(path):
|
||||||
|
continue
|
||||||
|
if kind == "talk engine port" and PORT_SKIP.search(path):
|
||||||
|
continue
|
||||||
m = rx.search(text)
|
m = rx.search(text)
|
||||||
if m:
|
if m:
|
||||||
hits.append((kind, m.group(0).strip()[:60]))
|
hits.append((kind, m.group(0).strip()[:60]))
|
||||||
@@ -150,9 +221,11 @@ def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict], *, line_fn=Non
|
|||||||
# Other guards (ci_hygiene) reuse this walker with their own line rules.
|
# Other guards (ci_hygiene) reuse this walker with their own line rules.
|
||||||
line_fn = line_fn or scan_line
|
line_fn = line_fn or scan_line
|
||||||
path_ok = path_ok or _default_path_ok
|
path_ok = path_ok or _default_path_ok
|
||||||
pre = prefilter or "|".join([re.escape(h) for h in HOSTS] + KEYS + [
|
pre = prefilter or "|".join([re.escape(h) for h in HOSTS + VOICE_HOSTS] + KEYS + VOICE_KEYS + [
|
||||||
"anthropic", "openai", "groq", "mistral", "generativeai", "genai", "cohere",
|
"anthropic", "openai", "groq", "mistral", "generativeai", "genai", "cohere",
|
||||||
"together", "cerebras", "litellm"])
|
"together", "cerebras", "litellm", "deepgram", "elevenlabs", "cartesia", "play\\.ht", "resemble",
|
||||||
|
"heygen", "googleapis\\.com", "amazonaws\\.com", "api\\.cloudflare\\.com", ":8791", ":8788",
|
||||||
|
":8794", "%3[aA]87", r"^\s*\[ai\]", '"ai"'])
|
||||||
try:
|
try:
|
||||||
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
@@ -209,7 +282,8 @@ def parse_added(repo: str, diff: str, allow: list[dict], *, line_fn=None, path_o
|
|||||||
# ---- cache: a tree scan runs once per (repo, sha, rules+allow) --------------
|
# ---- cache: a tree scan runs once per (repo, sha, rules+allow) --------------
|
||||||
def _fingerprint(allow: list[dict]) -> str:
|
def _fingerprint(allow: list[dict]) -> str:
|
||||||
return hashlib.sha256(
|
return hashlib.sha256(
|
||||||
json.dumps([HOSTS, KEYS, PY_SDKS, JS_SDKS, SKIP.pattern, allow], sort_keys=True).encode()
|
json.dumps([HOSTS, KEYS, VOICE_HOSTS, VOICE_KEYS, [r.pattern for _, r in RULES], PY_SDKS, JS_SDKS,
|
||||||
|
SKIP.pattern, allow], sort_keys=True, default=str).encode()
|
||||||
).hexdigest()[:16]
|
).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
@@ -253,7 +327,8 @@ def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> li
|
|||||||
allow = load_allow()
|
allow = load_allow()
|
||||||
fp = _fingerprint(allow)
|
fp = _fingerprint(allow)
|
||||||
if is_default_head:
|
if is_default_head:
|
||||||
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
return cached_scan(f"tree:{repo}:{sha}:{fp}",
|
||||||
|
lambda: [f for f in scan_tree(repo, bare, sha, allow) if f.kind not in WARN_ONLY_KINDS])
|
||||||
return cached_scan(
|
return cached_scan(
|
||||||
f"pr:{repo}:{sha}:{fp}",
|
f"pr:{repo}:{sha}:{fp}",
|
||||||
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||||
@@ -268,6 +343,18 @@ def status_for(findings: list[Finding], whole_tree: bool,
|
|||||||
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
||||||
09-23: his desktop work is never blocked by us)."""
|
09-23: his desktop work is never blocked by us)."""
|
||||||
scope = "in tree" if whole_tree else "added"
|
scope = "in tree" if whole_tree else "added"
|
||||||
|
soft = [f for f in findings if f.kind in WARN_ONLY_KINDS]
|
||||||
|
findings = [f for f in findings if f.kind not in WARN_ONLY_KINDS]
|
||||||
|
if not findings and not grant and soft:
|
||||||
|
f = soft[0]
|
||||||
|
return "success", f"⚠ WARN: new Veron Ollama ref {f.path}:{f.line}. {OLLAMA_MSG}"[:140], f
|
||||||
|
rolling = [f for f in findings if f.kind in SOFT_KINDS]
|
||||||
|
if findings and len(rolling) == len(findings) and not grant:
|
||||||
|
f, n = rolling[0], len(rolling)
|
||||||
|
return "success", (f"⚠ WARN (rolling out, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
||||||
|
f"{scope}, e.g. {f.path}:{f.line} {f.match}")[:140], f
|
||||||
|
if rolling:
|
||||||
|
findings = [f for f in findings if f.kind not in SOFT_KINDS]
|
||||||
if not findings and grant:
|
if not findings and grant:
|
||||||
g, n = grant[0], len(grant)
|
g, n = grant[0], len(grant)
|
||||||
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
||||||
@@ -286,6 +373,12 @@ def status_for(findings: list[Finding], whole_tree: bool,
|
|||||||
|
|
||||||
def report(repos: list[str]) -> int:
|
def report(repos: list[str]) -> int:
|
||||||
allow = load_allow()
|
allow = load_allow()
|
||||||
|
for e in OVERCAP:
|
||||||
|
print(f"## OVER-CAP exemption (> {MAX_DAYS} days, NOT applied): {e['repo']} {e['paths']} "
|
||||||
|
f"[{e['exemption']}] expires {e['expires']}")
|
||||||
|
for e in EXPIRED:
|
||||||
|
print(f"## EXPIRED exemption (no longer excuses anything): {e['repo']} {e['paths']} "
|
||||||
|
f"[{e['exemption']}] expired {e['expires']}")
|
||||||
total = 0
|
total = 0
|
||||||
for repo in repos:
|
for repo in repos:
|
||||||
bare = WORK / f"{repo}.git"
|
bare = WORK / f"{repo}.git"
|
||||||
|
|||||||
23
scripts/drill_cross_host.sh
Executable file
23
scripts/drill_cross_host.sh
Executable file
@@ -0,0 +1,23 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed.
|
||||||
|
# Usage: bash drill_cross_host.sh (needs lockbox keys RESTIC_WINDYGIT_PASSWORD, WINDYGIT_R2_ACCESS_KEY_ID, WINDYGIT_R2_SECRET_ACCESS_KEY, WINDYGIT_R2_ENDPOINT)
|
||||||
|
set -euo pipefail
|
||||||
|
umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX)
|
||||||
|
trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT
|
||||||
|
lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null
|
||||||
|
lockbox-get WINDYGIT_R2_ACCESS_KEY_ID "$W/ak" >/dev/null; lockbox-get WINDYGIT_R2_SECRET_ACCESS_KEY "$W/sk" >/dev/null; lockbox-get WINDYGIT_R2_ENDPOINT "$W/ep" >/dev/null
|
||||||
|
export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")"
|
||||||
|
export RESTIC_REPOSITORY="s3:$(cat "$W/ep")/windy-git-backups/restic"
|
||||||
|
restic snapshots --tag windygit-state --compact | tail -3
|
||||||
|
restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet
|
||||||
|
ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}'
|
||||||
|
docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null
|
||||||
|
for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done
|
||||||
|
for db in gitea windygit; do
|
||||||
|
docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db"
|
||||||
|
docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true
|
||||||
|
done
|
||||||
|
for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do
|
||||||
|
echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")"
|
||||||
|
done
|
||||||
|
echo "cross-host drill OK (cleaned up)"
|
||||||
@@ -5,10 +5,10 @@ set -euo pipefail
|
|||||||
here=$(cd "$(dirname "$0")" && pwd)
|
here=$(cd "$(dirname "$0")" && pwd)
|
||||||
dest="$HOME/.local/share/secret-tools"
|
dest="$HOME/.local/share/secret-tools"
|
||||||
mkdir -p "$dest" "$HOME/.local/bin"
|
mkdir -p "$dest" "$HOME/.local/bin"
|
||||||
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$dest/"
|
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$here/lockbox_names.py" "$dest/"
|
||||||
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py"; do
|
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py" "lockbox-names:lockbox_names.py"; do
|
||||||
n=${pair%%:*}; f=${pair##*:}
|
n=${pair%%:*}; f=${pair##*:}
|
||||||
printf '#!/usr/bin/env bash\nexec python3 "%s/%s" "$@"\n' "$dest" "$f" > "$HOME/.local/bin/$n"
|
printf '#!/usr/bin/env bash\nexec python3 "%s/%s" "$@"\n' "$dest" "$f" > "$HOME/.local/bin/$n"
|
||||||
chmod 755 "$HOME/.local/bin/$n"
|
chmod 755 "$HOME/.local/bin/$n"
|
||||||
done
|
done
|
||||||
echo "installed secret-scan, env-names and lockbox-put (shapes from secret_shapes.py, same as secret-guard)"
|
echo "installed secret-scan, env-names, lockbox-put and lockbox-names (shapes from secret_shapes.py, same as secret-guard)"
|
||||||
|
|||||||
134
scripts/lockbox_names.py
Normal file
134
scripts/lockbox_names.py
Normal file
@@ -0,0 +1,134 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""lockbox-names: DISCOVER what the lockbox holds without reading it (Boss rule 10-01).
|
||||||
|
|
||||||
|
lockbox-names [REGEX] (case-insensitive; matches the section heading or the label)
|
||||||
|
|
||||||
|
Prints one row per entry: SECTION HEADING | LABEL | kind | resolvable
|
||||||
|
kind env = `KEY=value` line md = `- **`KEY`**: `value`` line
|
||||||
|
label = a bold prose label (`**Password (X):** ...`) file = secrets/**/*.env key
|
||||||
|
resolvable yes = `lockbox-get LABEL FILE` returns exactly one value
|
||||||
|
dup = defined with 2+ different values (lockbox-get refuses)
|
||||||
|
no = a prose-only label, or not an exact key
|
||||||
|
|
||||||
|
NEVER prints a value or any prose after a label. A label or heading that itself looks
|
||||||
|
like a secret (secret_shapes) is replaced by <secret-shaped>. Reads the COMMITTED lockbox at
|
||||||
|
origin/main (like lockbox-get; LOCKBOX_REF=<ref> or WORKTREE overrides). Memory only, stdout only.
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import secret_shapes as ss # noqa: E402
|
||||||
|
|
||||||
|
REPO = os.environ.get("LOCKBOX_REPO", os.path.expanduser("~/kit-army-config"))
|
||||||
|
REF = os.environ.get("LOCKBOX_REF", "origin/main")
|
||||||
|
HEAD = re.compile(r"^#{1,6}\s+(.*\S)\s*$")
|
||||||
|
ENV = re.compile(r"^([A-Z][A-Z0-9_]{2,})=(.*)$")
|
||||||
|
MD = re.compile(r"^\s*[-*]?\s*\*\*`([A-Za-z0-9_]+)`\*\*\s*:\s*`([^`]+)`")
|
||||||
|
LABEL = re.compile(r"\*\*([^*`]{2,70}?)\*\*")
|
||||||
|
|
||||||
|
|
||||||
|
def git(*a: str) -> subprocess.CompletedProcess:
|
||||||
|
return subprocess.run(["git", "-C", REPO, *a], capture_output=True, text=True, errors="ignore")
|
||||||
|
|
||||||
|
|
||||||
|
def read_sources() -> dict[str, str]:
|
||||||
|
"""{path: text} for ACCESS_LOCKBOX.md and secrets/**/*.env."""
|
||||||
|
if REF == "WORKTREE":
|
||||||
|
out = {}
|
||||||
|
for p in [Path(REPO, "ACCESS_LOCKBOX.md"), *Path(REPO, "secrets").rglob("*.env")]:
|
||||||
|
if p.is_file():
|
||||||
|
out[str(p.relative_to(REPO))] = p.read_text(errors="ignore")
|
||||||
|
return out
|
||||||
|
if REF.startswith("origin/"):
|
||||||
|
git("fetch", "-q", "origin", REF.split("/", 1)[1])
|
||||||
|
names = ["ACCESS_LOCKBOX.md"] + [
|
||||||
|
p for p in git("ls-tree", "-r", "--name-only", REF, "--", "secrets").stdout.splitlines() if p.endswith(".env")]
|
||||||
|
out = {}
|
||||||
|
for n in names:
|
||||||
|
r = git("show", f"{REF}:{n}")
|
||||||
|
if r.returncode == 0:
|
||||||
|
out[n] = r.stdout
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def safe(text: str, limit: int = 70) -> str:
|
||||||
|
text = re.sub(r"\s+", " ", text).strip()
|
||||||
|
return "<secret-shaped>" if ss.find(text) else text[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def h(v: str) -> str:
|
||||||
|
return hashlib.sha256(v.strip().strip('"').strip("'").encode()).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def collect(src: dict[str, str]):
|
||||||
|
"""(rows, values) where values[KEY] = {hash,...} for resolvability; nothing printed from it."""
|
||||||
|
rows, values = [], {}
|
||||||
|
for path, text in src.items():
|
||||||
|
section = path
|
||||||
|
for line in text.splitlines():
|
||||||
|
m = HEAD.match(line) if path.endswith(".md") else None
|
||||||
|
if m:
|
||||||
|
section = safe(m.group(1), 90)
|
||||||
|
continue
|
||||||
|
m = ENV.match(line)
|
||||||
|
if m:
|
||||||
|
values.setdefault(m.group(1), set()).add(h(m.group(2)))
|
||||||
|
rows.append((section, m.group(1), "file" if path.startswith("secrets/") else "env"))
|
||||||
|
continue
|
||||||
|
m = MD.match(line)
|
||||||
|
if m:
|
||||||
|
values.setdefault(m.group(1), set()).add(h(m.group(2)))
|
||||||
|
rows.append((section, m.group(1), "md"))
|
||||||
|
continue
|
||||||
|
if path.endswith(".md"):
|
||||||
|
mm = LABEL.search(line)
|
||||||
|
if mm and not mm.group(1).startswith("http"):
|
||||||
|
rows.append((section, safe(mm.group(1)), "label"))
|
||||||
|
return rows, values
|
||||||
|
|
||||||
|
|
||||||
|
def resolvable(label: str, kind: str, values) -> str:
|
||||||
|
if kind not in ("env", "md", "file"):
|
||||||
|
return "no"
|
||||||
|
n = len(values.get(label, ()))
|
||||||
|
return "yes" if n == 1 else "dup" if n > 1 else "no"
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
argv = list(sys.argv[1:] if argv is None else argv)
|
||||||
|
rx = re.compile(argv[0], re.I) if argv else None
|
||||||
|
src = read_sources()
|
||||||
|
if not src:
|
||||||
|
print("lockbox-names: cannot read the lockbox")
|
||||||
|
return 2
|
||||||
|
rows, values = collect(src)
|
||||||
|
seen, n = set(), 0
|
||||||
|
for section, label, kind in rows:
|
||||||
|
if rx and not (rx.search(section) or rx.search(label)):
|
||||||
|
continue
|
||||||
|
key = (section, label, kind)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
n += 1
|
||||||
|
print(f"{section} | {label} | {kind} | {resolvable(label, kind, values)}")
|
||||||
|
print(f"# {n} entr{'y' if n == 1 else 'ies'}; names only, values never printed")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
try:
|
||||||
|
sys.exit(main())
|
||||||
|
except re.error:
|
||||||
|
print("lockbox-names: bad regex")
|
||||||
|
sys.exit(2)
|
||||||
|
except Exception as e: # never a traceback
|
||||||
|
print(f"lockbox-names: error: {type(e).__name__}")
|
||||||
|
sys.exit(2)
|
||||||
@@ -54,7 +54,7 @@ REPOS = os.environ.get(
|
|||||||
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
||||||
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
||||||
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
||||||
" windy-inbox windy-text windy-call windy-cell",
|
" windy-inbox windy-text windy-call windy-cell windy-hand-site windy-calendar-site",
|
||||||
).split()
|
).split()
|
||||||
|
|
||||||
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
||||||
|
|||||||
209
scripts/runner_guard.py
Normal file
209
scripts/runner_guard.py
Normal file
@@ -0,0 +1,209 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Runner guard: no workflow may let a STRANGER's code reach a self-hosted runner (Boss 10-01).
|
||||||
|
|
||||||
|
Our self-hosted GitHub runners run on Veron as `github-runner`, which is in the docker group
|
||||||
|
(= root on Veron). Until ephemeral containerised runners exist (after launch), the cheap
|
||||||
|
guard is to refuse the workflow shapes that hand a self-hosted runner to outsiders:
|
||||||
|
|
||||||
|
R1 pull_request_target : runs with secrets/write token in the BASE repo context
|
||||||
|
R2 pull_request on self-hosted : fork PRs run their own code, unless the job is gated to
|
||||||
|
same-repo heads (`if:` on head.repo.full_name == github.repository
|
||||||
|
or head.repo.fork == false) or an `environment:`
|
||||||
|
R3 issue_comment / workflow_run / issues / discussion* / pull_request_review* / fork / watch
|
||||||
|
: anyone can fire these; never on self-hosted without an environment gate
|
||||||
|
(push, tags, schedule, workflow_dispatch, repository_dispatch, workflow_call: writers only, fine)
|
||||||
|
|
||||||
|
A job counts as self-hosted when its runs-on names `self-hosted`, or is an expression we
|
||||||
|
can't resolve (conservative). Findings carry file:line, never file content beyond that.
|
||||||
|
|
||||||
|
python3 scripts/runner_guard.py lint FILE... # local files
|
||||||
|
python3 scripts/runner_guard.py report [--owners a,b] # default branch of every PUBLIC repo
|
||||||
|
python3 scripts/runner_guard.py pr [--owners a,b] [--post] # open PRs on public repos: changed workflows
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import base64
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
OWNERS = ["sneakyfree", "VERONTECH", "Windstorm-Institute", "Windstorm-Labs", "Public-Streamer"]
|
||||||
|
CTX = "windy-git/runner-guard"
|
||||||
|
OUTSIDE = {"issue_comment", "workflow_run", "issues", "discussion", "discussion_comment",
|
||||||
|
"pull_request_review", "pull_request_review_comment", "fork", "watch"}
|
||||||
|
SAME_REPO_GATES = ("head.repo.full_name == github.repository", "github.repository == github.event.pull_request.head.repo.full_name",
|
||||||
|
"head.repo.fork == false", "!github.event.pull_request.head.repo.fork")
|
||||||
|
|
||||||
|
|
||||||
|
def _node_map(node):
|
||||||
|
"""{key: (value_node, line)} for a YAML mapping node."""
|
||||||
|
if not isinstance(node, yaml.MappingNode):
|
||||||
|
return {}
|
||||||
|
return {k.value: (v, k.start_mark.line + 1) for k, v in node.value if isinstance(k, yaml.ScalarNode)}
|
||||||
|
|
||||||
|
|
||||||
|
def _triggers(on_node) -> dict[str, int]:
|
||||||
|
"""{event: line}."""
|
||||||
|
if isinstance(on_node, yaml.ScalarNode):
|
||||||
|
return {on_node.value: on_node.start_mark.line + 1}
|
||||||
|
if isinstance(on_node, yaml.SequenceNode):
|
||||||
|
return {n.value: n.start_mark.line + 1 for n in on_node.value if isinstance(n, yaml.ScalarNode)}
|
||||||
|
return {k: line for k, (_v, line) in _node_map(on_node).items()}
|
||||||
|
|
||||||
|
|
||||||
|
def _self_hosted(runs_on) -> bool:
|
||||||
|
if runs_on is None:
|
||||||
|
return False
|
||||||
|
text = yaml.serialize(runs_on) if isinstance(runs_on, yaml.Node) else str(runs_on)
|
||||||
|
return "self-hosted" in text or "${{" in text
|
||||||
|
|
||||||
|
|
||||||
|
def lint_text(path: str, text: str) -> list[tuple[str, int, str, str]]:
|
||||||
|
"""[(path, line, rule, message)]. Unparseable YAML is a finding (it cannot be reviewed)."""
|
||||||
|
try:
|
||||||
|
root = yaml.compose(text)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
line = getattr(getattr(e, "problem_mark", None), "line", 0) + 1
|
||||||
|
return [(path, line, "R0", "workflow YAML does not parse; cannot be checked")]
|
||||||
|
top = _node_map(root)
|
||||||
|
if "on" not in top or "jobs" not in top:
|
||||||
|
return []
|
||||||
|
trig = _triggers(top["on"][0])
|
||||||
|
jobs = _node_map(top["jobs"][0])
|
||||||
|
out = []
|
||||||
|
if "pull_request_target" in trig:
|
||||||
|
out.append((path, trig["pull_request_target"], "R1",
|
||||||
|
"pull_request_target runs fork code with base-repo secrets; not allowed"))
|
||||||
|
for name, (jnode, jline) in jobs.items():
|
||||||
|
j = _node_map(jnode)
|
||||||
|
ro = j.get("runs-on", (None, jline))
|
||||||
|
if not _self_hosted(ro[0]):
|
||||||
|
continue
|
||||||
|
has_env = "environment" in j
|
||||||
|
cond = j["if"][0].value if "if" in j and isinstance(j["if"][0], yaml.ScalarNode) else ""
|
||||||
|
same_repo = any(g in cond.replace(" ", " ") for g in SAME_REPO_GATES)
|
||||||
|
if "pull_request" in trig and not (has_env or same_repo):
|
||||||
|
out.append((path, ro[1], "R2", f"job '{name}' runs fork PR code on a self-hosted runner "
|
||||||
|
"(gate it: if: github.event.pull_request.head.repo.full_name == github.repository, or an environment)"))
|
||||||
|
for ev in sorted(OUTSIDE & trig.keys()):
|
||||||
|
if not has_env:
|
||||||
|
out.append((path, trig[ev], "R3", f"'{ev}' can be fired by anyone and job '{name}' is self-hosted "
|
||||||
|
"without an environment gate"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- GitHub side
|
||||||
|
def gh(*args: str, check=True) -> str:
|
||||||
|
r = subprocess.run(["gh", "api", *args], capture_output=True, text=True, timeout=60)
|
||||||
|
if check and r.returncode != 0:
|
||||||
|
raise RuntimeError(f"gh api {args[0]} failed")
|
||||||
|
return r.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def public_repos(owners) -> list[tuple[str, str]]:
|
||||||
|
out = []
|
||||||
|
for o in owners:
|
||||||
|
txt = gh(f"users/{o}/repos?per_page=100&type=owner", "--paginate",
|
||||||
|
"--jq", '.[]|select(.private==false and .archived==false)|.full_name+" "+.default_branch', check=False)
|
||||||
|
out += [tuple(row.split()) for row in txt.splitlines() if row.strip()]
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def workflows_at(full: str, ref: str) -> list[tuple[str, str]]:
|
||||||
|
txt = gh(f"repos/{full}/contents/.github/workflows?ref={ref}", "--jq",
|
||||||
|
'.[]|select(.type=="file")|.path', check=False)
|
||||||
|
files = [p for p in txt.splitlines() if p.endswith((".yml", ".yaml"))]
|
||||||
|
return [(p, file_at(full, p, ref)) for p in files]
|
||||||
|
|
||||||
|
|
||||||
|
def file_at(full: str, path: str, ref: str) -> str:
|
||||||
|
raw = gh(f"repos/{full}/contents/{path}?ref={ref}", "--jq", ".content", check=False).strip()
|
||||||
|
return base64.b64decode(raw).decode("utf-8", "replace") if raw else ""
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_report(owners) -> int:
|
||||||
|
hits = 0
|
||||||
|
repos = public_repos(owners)
|
||||||
|
for full, branch in repos:
|
||||||
|
for path, text in workflows_at(full, branch):
|
||||||
|
for p, line, rule, msg in lint_text(path, text):
|
||||||
|
hits += 1
|
||||||
|
print(f"{full}\t{p}:{line}\t{rule}\t{msg}")
|
||||||
|
print(f"# runner-guard sweep: {hits} hit(s) in {len(repos)} public repos")
|
||||||
|
return 1 if hits else 0
|
||||||
|
|
||||||
|
|
||||||
|
STATE = os.environ.get("RUNNER_GUARD_STATE", "/var/lib/windy-git/runner-guard-posted.json")
|
||||||
|
|
||||||
|
|
||||||
|
def cmd_pr(owners, post: bool) -> int:
|
||||||
|
# Post each (repo, sha, state, description) ONCE: the sync runs every 5 min and GitHub caps
|
||||||
|
# statuses per sha+context at 1000.
|
||||||
|
try:
|
||||||
|
with open(STATE) as fh:
|
||||||
|
posted = set(json.load(fh))
|
||||||
|
except (OSError, ValueError):
|
||||||
|
posted = set()
|
||||||
|
seen = set()
|
||||||
|
for full, _branch in public_repos(owners):
|
||||||
|
prs = gh(f"repos/{full}/pulls?state=open&per_page=50", "--jq",
|
||||||
|
'.[]|(.number|tostring)+" "+.head.sha+" "+.head.repo.full_name', check=False)
|
||||||
|
for line in prs.splitlines():
|
||||||
|
num, sha, head_repo = line.split(" ", 2)
|
||||||
|
files = gh(f"repos/{full}/pulls/{num}/files?per_page=100", "--jq",
|
||||||
|
'.[]|select(.status!="removed")|.filename', check=False).split()
|
||||||
|
wf = [f for f in files if f.startswith(".github/workflows/") and f.endswith((".yml", ".yaml"))]
|
||||||
|
# a fork's own content is read from the head repo at the head sha
|
||||||
|
src = head_repo if head_repo and head_repo != "null" else full
|
||||||
|
found = [h for f in wf for h in lint_text(f, file_at(src, f, sha))]
|
||||||
|
if found:
|
||||||
|
p, ln, rule, msg = found[0]
|
||||||
|
state, desc = "failure", f"BLOCKED: {rule} {p}:{ln}: {msg}"[:140]
|
||||||
|
else:
|
||||||
|
state, desc = "success", ("OK: no workflow changes" if not wf else
|
||||||
|
"OK: no stranger-code path to a self-hosted runner")
|
||||||
|
key = f"{full}@{sha}:{state}:{desc}"
|
||||||
|
seen.add(key)
|
||||||
|
if key in posted:
|
||||||
|
continue
|
||||||
|
print(f"{full}#{num}@{sha[:7]} {state} {desc}")
|
||||||
|
if post:
|
||||||
|
gh(f"repos/{full}/statuses/{sha}", "-f", f"state={state}", "-f", f"context={CTX}",
|
||||||
|
"-f", f"description={desc}", check=False)
|
||||||
|
posted.add(key)
|
||||||
|
if post: # keep only keys for PRs still open, so the file never grows without bound
|
||||||
|
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
||||||
|
with open(STATE, "w") as fh:
|
||||||
|
json.dump(sorted(posted & seen), fh)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None) -> int:
|
||||||
|
ap = argparse.ArgumentParser(prog="runner_guard")
|
||||||
|
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||||
|
lint_p = sub.add_parser("lint")
|
||||||
|
lint_p.add_argument("files", nargs="+")
|
||||||
|
rep = sub.add_parser("report")
|
||||||
|
rep.add_argument("--owners", default=",".join(OWNERS))
|
||||||
|
prp = sub.add_parser("pr")
|
||||||
|
prp.add_argument("--owners", default="sneakyfree") # self-hosted runners exist only there
|
||||||
|
prp.add_argument("--post", action="store_true")
|
||||||
|
a = ap.parse_args(argv)
|
||||||
|
if a.cmd == "lint":
|
||||||
|
hits = []
|
||||||
|
for f in a.files:
|
||||||
|
with open(f, errors="replace") as fh:
|
||||||
|
hits += lint_text(f, fh.read())
|
||||||
|
for p_, ln, rule, msg in hits:
|
||||||
|
print(f"{p_}:{ln}\t{rule}\t{msg}")
|
||||||
|
return 1 if hits else 0
|
||||||
|
owners = a.owners.split(",")
|
||||||
|
return cmd_report(owners) if a.cmd == "report" else cmd_pr(owners, a.post)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -23,13 +23,14 @@ PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|||||||
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
||||||
("32-hex secret assignment", re.compile(
|
("32-hex secret assignment", re.compile(
|
||||||
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
||||||
|
("pypi token", re.compile(r"\bpypi-AgE[A-Za-z0-9_-]{50,}")),
|
||||||
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
||||||
]
|
]
|
||||||
|
|
||||||
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
||||||
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
||||||
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
||||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}")
|
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}|pypi-AgE")
|
||||||
|
|
||||||
|
|
||||||
def h8(value: str | bytes) -> str:
|
def h8(value: str | bytes) -> str:
|
||||||
|
|||||||
@@ -38,7 +38,7 @@ FAILED=0
|
|||||||
|
|
||||||
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
||||||
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
||||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell}"
|
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell windy-hand-site windy-calendar-site}"
|
||||||
|
|
||||||
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
||||||
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
||||||
@@ -94,6 +94,11 @@ if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then
|
|||||||
log "FAILED pr status bridge"; FAILED=1
|
log "FAILED pr status bridge"; FAILED=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Runner guard (Boss 10-01): PUBLIC sneakyfree repos have self-hosted GitHub runners on Veron.
|
||||||
|
# A PR that changes a workflow so a stranger's code could reach one gets a red
|
||||||
|
# windy-git/runner-guard status. Each status is posted once; never fails the sync.
|
||||||
|
timeout -k 10 120 python3 "$(dirname "$0")/runner_guard.py" pr --post || log "runner-guard failed or timed out (non-fatal)"
|
||||||
|
|
||||||
# CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
|
# CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
|
||||||
# Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
|
# Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
|
||||||
timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)"
|
timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)"
|
||||||
|
|||||||
Reference in New Issue
Block a user