Compare commits
8 Commits
veron-olla
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f19e23eaf | ||
| ef96051d6f | |||
|
|
a0dc6f8568 | ||
|
|
51e0b9d30b | ||
|
|
51777b0ad0 | ||
|
|
cbcb4c206b | ||
|
|
2c62e2834a | ||
| 3a9b7ecab7 |
64
api/tests/test_lockbox_names.py
Normal file
64
api/tests/test_lockbox_names.py
Normal file
@@ -0,0 +1,64 @@
|
||||
"""lockbox-names: headings + labels + resolvable, NEVER a value or prose after a label."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
V1 = "Qm7xT2vLp9RkZw4HnB8dYc3S" # synthetic values
|
||||
V2 = "other-fake-value-1234567890"
|
||||
V3 = "dup-one-aaaaaaaaaaaaaaaa"
|
||||
V4 = "dup-two-bbbbbbbbbbbbbbbb"
|
||||
PROSE = "ZZPROSEZZ-not-for-printing"
|
||||
|
||||
|
||||
def make(tmp_path):
|
||||
r = tmp_path / "kit"
|
||||
(r / "secrets" / "x").mkdir(parents=True)
|
||||
(r / "ACCESS_LOCKBOX.md").write_text(
|
||||
"# LOCKBOX\n\n## 🔷 AZURE signing (added 10-01)\n"
|
||||
f"- **Tenant:** {PROSE} lives in the portal\n"
|
||||
f"- **`AZURE_CLIENT_ID`**: `{V1}`\n"
|
||||
f"- **Secret (AZURE_CLIENT_SECRET):** `{V2}`\n"
|
||||
"## GOOGLE oauth\n"
|
||||
f"GOOGLE_OAUTH_CLIENT_ID={V2}\n"
|
||||
f"- **`DUP_KEY`**: `{V3}`\n- **`DUP_KEY`**: `{V4}`\n"
|
||||
f"## stray\n**{V1}** is a heading-like bold that is secret shaped? no, just label\n")
|
||||
(r / "secrets" / "x" / "a.env").write_text(f"FILE_KEY={V1}\n")
|
||||
subprocess.run(["git", "init", "-q", "-b", "main"], cwd=r, check=True)
|
||||
return r
|
||||
|
||||
|
||||
def run(r, *args):
|
||||
e = {**os.environ, "LOCKBOX_REPO": str(r), "LOCKBOX_REF": "WORKTREE"}
|
||||
p = subprocess.run([sys.executable, str(ROOT / "scripts" / "lockbox_names.py"), *args],
|
||||
capture_output=True, text=True, env=e)
|
||||
return p.returncode, p.stdout + p.stderr
|
||||
|
||||
|
||||
def test_lists_labels_and_resolvable_without_values_or_prose(tmp_path):
|
||||
r = make(tmp_path)
|
||||
rc, out = run(r, "AZURE|GOOGLE|FILE|DUP")
|
||||
assert rc == 0
|
||||
assert "AZURE signing (added 10-01) | AZURE_CLIENT_ID | md | yes" in out
|
||||
assert "| GOOGLE_OAUTH_CLIENT_ID | env | yes" in out
|
||||
assert "| FILE_KEY | file | yes" in out
|
||||
assert "| DUP_KEY | md | dup" in out
|
||||
# a prose label is listed but never resolvable, and nothing after the label leaks
|
||||
assert "| Tenant: " not in out or "| Tenant" in out
|
||||
assert "| label | no" in out
|
||||
for secret in (V1, V2, V3, V4, PROSE, "lives in the portal"):
|
||||
assert secret not in out
|
||||
for i in range(0, len(secret) - 7):
|
||||
assert secret[i:i + 8] not in out
|
||||
|
||||
|
||||
def test_filter_and_bad_regex(tmp_path):
|
||||
r = make(tmp_path)
|
||||
rc, out = run(r, "NOSUCHTHING")
|
||||
assert rc == 0 and "0 entries" in out
|
||||
rc, out = run(r, "(")
|
||||
assert rc == 2 and "bad regex" in out
|
||||
70
api/tests/test_runner_guard.py
Normal file
70
api/tests/test_runner_guard.py
Normal file
@@ -0,0 +1,70 @@
|
||||
"""runner-guard: workflow shapes that hand a self-hosted runner to strangers."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
_spec = importlib.util.spec_from_file_location("runner_guard", ROOT / "scripts" / "runner_guard.py")
|
||||
rg = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["runner_guard"] = rg
|
||||
_spec.loader.exec_module(rg)
|
||||
|
||||
|
||||
def rules(text):
|
||||
return [(r, ln) for _p, ln, r, _m in rg.lint_text("w.yml", text)]
|
||||
|
||||
|
||||
def test_pull_request_target_always_fails():
|
||||
assert rules("on: pull_request_target\njobs:\n a:\n runs-on: ubuntu-latest\n steps: []\n")[0][0] == "R1"
|
||||
|
||||
|
||||
def test_fork_pr_on_self_hosted_fails_and_points_at_runs_on():
|
||||
wf = "on:\n pull_request:\njobs:\n t:\n runs-on: [self-hosted, linux, x64]\n steps: []\n"
|
||||
assert rules(wf) == [("R2", 5)]
|
||||
|
||||
|
||||
def test_same_repo_gate_or_environment_passes():
|
||||
gated = ("on: [pull_request]\njobs:\n t:\n if: github.event.pull_request.head.repo.full_name == github.repository\n"
|
||||
" runs-on: [self-hosted]\n steps: []\n")
|
||||
env = "on: [pull_request]\njobs:\n t:\n environment: ci\n runs-on: self-hosted\n steps: []\n"
|
||||
assert rules(gated) == [] and rules(env) == []
|
||||
|
||||
|
||||
def test_outsider_events_on_self_hosted_fail_but_writer_events_pass():
|
||||
wf = "on:\n issue_comment:\n workflow_run:\n workflows: [x]\njobs:\n t:\n runs-on: self-hosted\n steps: []\n"
|
||||
assert sorted(r for r, _ in rules(wf)) == ["R3", "R3"]
|
||||
ok = "on:\n push:\n tags: ['v*']\n workflow_dispatch:\n schedule:\n - cron: '0 3 * * *'\njobs:\n t:\n runs-on: self-hosted\n steps: []\n"
|
||||
assert rules(ok) == []
|
||||
|
||||
|
||||
def test_expression_runs_on_is_treated_as_self_hosted_and_hosted_runner_is_fine():
|
||||
expr = "on: pull_request\njobs:\n t:\n runs-on: ${{ matrix.os }}\n steps: []\n"
|
||||
hosted = "on: pull_request\njobs:\n t:\n runs-on: ubuntu-latest\n steps: []\n"
|
||||
assert rules(expr) == [("R2", 4)] and rules(hosted) == []
|
||||
|
||||
|
||||
def test_broken_yaml_is_a_finding_and_non_workflows_are_ignored():
|
||||
assert rules("on: [push\njobs: {")[0][0] == "R0"
|
||||
assert rules("name: just a file\n") == []
|
||||
|
||||
|
||||
def test_pr_mode_posts_each_status_once(monkeypatch, tmp_path):
|
||||
calls = []
|
||||
monkeypatch.setattr(rg, "STATE", str(tmp_path / "s.json"))
|
||||
monkeypatch.setattr(rg, "public_repos", lambda owners: [("o/r", "main")])
|
||||
monkeypatch.setattr(rg, "file_at", lambda *a: "on: push\\njobs:\\n t:\\n runs-on: self-hosted\\n steps: []\\n")
|
||||
|
||||
def fake_gh(*args, check=True):
|
||||
if args[0].startswith("repos/o/r/pulls?"):
|
||||
return "7 abc123 o/r\\n"
|
||||
if args[0].endswith("/files?per_page=100"):
|
||||
return ".github/workflows/ci.yml\\n"
|
||||
calls.append(args[0])
|
||||
return ""
|
||||
monkeypatch.setattr(rg, "gh", fake_gh)
|
||||
rg.cmd_pr(["o"], post=True)
|
||||
rg.cmd_pr(["o"], post=True)
|
||||
assert calls == ["repos/o/r/statuses/abc123"]
|
||||
@@ -57,3 +57,6 @@ allow:
|
||||
- repo: windytalk
|
||||
hashes: ["c4189d79"]
|
||||
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
||||
- repo: windy-agent
|
||||
hashes: ["84e0c0ea"]
|
||||
reason: "tests/test_log_redaction.py:56 Z.ai redaction fixture REPLACED by windy-agent #412 with a synthetic value; hash checked against the lockbox 10-01: not present."
|
||||
|
||||
28
deploy/windy0/nightly-runner-guard-sweep.sh
Executable file
28
deploy/windy0/nightly-runner-guard-sweep.sh
Executable file
@@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# Nightly (Boss 10-01): runner-guard over the default branch of EVERY public repo in Grant's
|
||||
# 5 GitHub accounts (runs on Veron: windy-git scripts/runner_guard.py report). Writes
|
||||
# ~/windy-orchestra/RUNNER_GUARD.md (repo, file:line, rule; no file content), appends ONE
|
||||
# BOARD line per NEW hit vs the last run, and prints "runner-guard sweep: N hit(s)" last, so
|
||||
# the windy-job heartbeat (--expect "runner-guard sweep: 0 hit") goes red while any hit exists.
|
||||
set -euo pipefail
|
||||
page=~/windy-orchestra/RUNNER_GUARD.md
|
||||
state=~/.local/state/runner-guard-sweep.txt
|
||||
mkdir -p "$(dirname "$state")"
|
||||
out=$(timeout 900 ssh -o BatchMode=yes -o ConnectTimeout=15 ts-veron \
|
||||
'cd /srv/windygit/src && timeout 850 python3 scripts/runner_guard.py report' || true)
|
||||
summary=$(grep '^# runner-guard sweep:' <<<"$out" || echo "# runner-guard sweep: ERROR (no summary)")
|
||||
hits=$(grep -v '^#' <<<"$out" | grep . || true)
|
||||
{
|
||||
echo "# Runner guard: stranger-code paths to self-hosted runners ($(date -u '+%Y-%m-%d %H:%MZ'))"
|
||||
echo "_Nightly; windy-git scripts/runner_guard.py. R1 pull_request_target · R2 fork PR on self-hosted without a same-repo/environment gate · R3 outsider events (issue_comment, workflow_run, ...) on self-hosted · R0 unparseable._"
|
||||
echo; echo "${summary#\# }"; echo
|
||||
echo "| repo | file:line | rule | fix |"; echo "|---|---|---|---|"
|
||||
while IFS=$'\t' read -r repo loc rule msg; do [[ -n $repo ]] && echo "| $repo | $loc | $rule | $msg |"; done <<<"$hits"
|
||||
} > "$page"
|
||||
new=$(comm -13 <(sort -u "$state" 2>/dev/null || true) <(cut -f1-3 <<<"$hits" | sort -u))
|
||||
cut -f1-3 <<<"$hits" | sort -u > "$state"
|
||||
if [[ -n "$new" ]]; then
|
||||
n=$(grep -c . <<<"$new")
|
||||
echo "$(date -u +%Y-%m-%dT%H:%MZ) Windy Git: 🚨 runner-guard: $n NEW stranger-code path(s) to a self-hosted runner in public repos; see ~/windy-orchestra/RUNNER_GUARD.md" >> ~/windy-orchestra/BOARD.md
|
||||
fi
|
||||
echo "${summary#\# }"
|
||||
7
deploy/windy0/windy-runner-guard-sweep.service
Normal file
7
deploy/windy0/windy-runner-guard-sweep.service
Normal file
@@ -0,0 +1,7 @@
|
||||
[Unit]
|
||||
Description=Nightly runner-guard sweep of every PUBLIC repo's workflows (Windy Git lane)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/windy-job windy-runner-guard-sweep 26h --expect "runner-guard sweep: 0 hit" --owner 13 -- %h/bin/nightly-runner-guard-sweep.sh
|
||||
TimeoutStartSec=1200
|
||||
9
deploy/windy0/windy-runner-guard-sweep.timer
Normal file
9
deploy/windy0/windy-runner-guard-sweep.timer
Normal file
@@ -0,0 +1,9 @@
|
||||
[Unit]
|
||||
Description=Nightly runner-guard sweep
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 09:40:00 UTC
|
||||
Persistent=true
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
31
docs/CUTOVER-PRIMARY.md
Normal file
31
docs/CUTOVER-PRIMARY.md
Normal file
@@ -0,0 +1,31 @@
|
||||
# Checklist: making Windy Git the PRIMARY home of one repo (after launch)
|
||||
|
||||
Status 2026-10-01: DRAFT, nothing flipped. GitHub stays the source of truth until Grant says otherwise.
|
||||
First candidate: `windy-git` itself (public, low blast radius). GitHub becomes the free off-site push-mirror.
|
||||
|
||||
## Preconditions (all must be true)
|
||||
- [x] Encrypted state backup (Postgres + Gitea config + repos) nightly, restore drill passed cross-host (10-01).
|
||||
- [x] Gitea on a patched release (1.24.7); upgrade path = snapshot first (docs/RESTORE-DRILL.md).
|
||||
- [ ] Heartbeat `windygit-state-backup` in heartbeats-veron expected list (asked Cloud/Super Admin 10-01).
|
||||
- [ ] A missed/failed backup PAGES (heartbeat 26h), tested once by skipping a night in a drill.
|
||||
- [ ] Boss/Grant capacity call on Veron (dedicated non-SMR volume for DB + git storage; root disk < 80%).
|
||||
- [ ] Post-launch freeze lifted (Hub). No cutover during store review or a launch window.
|
||||
|
||||
## Cutover for ONE repo (reversible at every step)
|
||||
1. Announce on BOARD; tell every consuming lane (no schema drift rule). Pause the sync for that repo only:
|
||||
remove it from `REPOS` in `scripts/sync_from_github.sh` FIRST (the sync force-overwrites Windy Git).
|
||||
2. Verify Windy Git main == GitHub main (sha equal), all branches/tags present, LFS (if any) in R2.
|
||||
3. Add a PUSH-mirror on the Windy Git repo -> GitHub (deploy key or scoped token, write on that repo only,
|
||||
interval 8h + on-commit), so GitHub keeps a current copy. Test with a throwaway branch.
|
||||
4. Flip the lanes' remote: `origin` = Windy Git (SSH/HTTPS via Windy SSO token), `github` = secondary.
|
||||
Lanes push to Windy Git only; the mirror carries it to GitHub.
|
||||
5. CI keeps running here (no change); remove the `pr_status_bridge` mirror-PR duplication for that repo
|
||||
(PRs are now native here; the bridge's GitHub status posting for it can stay for any open GitHub PRs).
|
||||
6. Watch 3 days: mirror lag, backup includes the new writes, no push rejected, no lane still pushing to GitHub.
|
||||
7. Rollback at any time: re-add the repo to `REPOS` (sync resumes GitHub->Windy Git, GitHub is intact via the
|
||||
push-mirror) and point lanes' remote back. Nothing is destroyed in either direction.
|
||||
|
||||
## NOT in scope for a first cutover
|
||||
Deploy workflows (stay disabled; deploys remain manual until a separate runner + scoped deploy keys exist),
|
||||
credential repos (soul/anima/kit-army-config), windy-pro (importer refuses by name), opening the forge to
|
||||
other members (Grant 09-23: registration closed; Hub 10-01: jit false, 4 conditions before any change).
|
||||
@@ -5,10 +5,10 @@ set -euo pipefail
|
||||
here=$(cd "$(dirname "$0")" && pwd)
|
||||
dest="$HOME/.local/share/secret-tools"
|
||||
mkdir -p "$dest" "$HOME/.local/bin"
|
||||
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$dest/"
|
||||
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py"; do
|
||||
cp "$here/secret_shapes.py" "$here/secret_scan.py" "$here/env_names.py" "$here/lockbox_put.py" "$here/lockbox_names.py" "$dest/"
|
||||
for pair in "secret-scan:secret_scan.py" "env-names:env_names.py" "lockbox-put:lockbox_put.py" "lockbox-names:lockbox_names.py"; do
|
||||
n=${pair%%:*}; f=${pair##*:}
|
||||
printf '#!/usr/bin/env bash\nexec python3 "%s/%s" "$@"\n' "$dest" "$f" > "$HOME/.local/bin/$n"
|
||||
chmod 755 "$HOME/.local/bin/$n"
|
||||
done
|
||||
echo "installed secret-scan, env-names and lockbox-put (shapes from secret_shapes.py, same as secret-guard)"
|
||||
echo "installed secret-scan, env-names, lockbox-put and lockbox-names (shapes from secret_shapes.py, same as secret-guard)"
|
||||
|
||||
134
scripts/lockbox_names.py
Normal file
134
scripts/lockbox_names.py
Normal file
@@ -0,0 +1,134 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lockbox-names: DISCOVER what the lockbox holds without reading it (Boss rule 10-01).
|
||||
|
||||
lockbox-names [REGEX] (case-insensitive; matches the section heading or the label)
|
||||
|
||||
Prints one row per entry: SECTION HEADING | LABEL | kind | resolvable
|
||||
kind env = `KEY=value` line md = `- **`KEY`**: `value`` line
|
||||
label = a bold prose label (`**Password (X):** ...`) file = secrets/**/*.env key
|
||||
resolvable yes = `lockbox-get LABEL FILE` returns exactly one value
|
||||
dup = defined with 2+ different values (lockbox-get refuses)
|
||||
no = a prose-only label, or not an exact key
|
||||
|
||||
NEVER prints a value or any prose after a label. A label or heading that itself looks
|
||||
like a secret (secret_shapes) is replaced by <secret-shaped>. Reads the COMMITTED lockbox at
|
||||
origin/main (like lockbox-get; LOCKBOX_REF=<ref> or WORKTREE overrides). Memory only, stdout only.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import secret_shapes as ss # noqa: E402
|
||||
|
||||
REPO = os.environ.get("LOCKBOX_REPO", os.path.expanduser("~/kit-army-config"))
|
||||
REF = os.environ.get("LOCKBOX_REF", "origin/main")
|
||||
HEAD = re.compile(r"^#{1,6}\s+(.*\S)\s*$")
|
||||
ENV = re.compile(r"^([A-Z][A-Z0-9_]{2,})=(.*)$")
|
||||
MD = re.compile(r"^\s*[-*]?\s*\*\*`([A-Za-z0-9_]+)`\*\*\s*:\s*`([^`]+)`")
|
||||
LABEL = re.compile(r"\*\*([^*`]{2,70}?)\*\*")
|
||||
|
||||
|
||||
def git(*a: str) -> subprocess.CompletedProcess:
|
||||
return subprocess.run(["git", "-C", REPO, *a], capture_output=True, text=True, errors="ignore")
|
||||
|
||||
|
||||
def read_sources() -> dict[str, str]:
|
||||
"""{path: text} for ACCESS_LOCKBOX.md and secrets/**/*.env."""
|
||||
if REF == "WORKTREE":
|
||||
out = {}
|
||||
for p in [Path(REPO, "ACCESS_LOCKBOX.md"), *Path(REPO, "secrets").rglob("*.env")]:
|
||||
if p.is_file():
|
||||
out[str(p.relative_to(REPO))] = p.read_text(errors="ignore")
|
||||
return out
|
||||
if REF.startswith("origin/"):
|
||||
git("fetch", "-q", "origin", REF.split("/", 1)[1])
|
||||
names = ["ACCESS_LOCKBOX.md"] + [
|
||||
p for p in git("ls-tree", "-r", "--name-only", REF, "--", "secrets").stdout.splitlines() if p.endswith(".env")]
|
||||
out = {}
|
||||
for n in names:
|
||||
r = git("show", f"{REF}:{n}")
|
||||
if r.returncode == 0:
|
||||
out[n] = r.stdout
|
||||
return out
|
||||
|
||||
|
||||
def safe(text: str, limit: int = 70) -> str:
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
return "<secret-shaped>" if ss.find(text) else text[:limit]
|
||||
|
||||
|
||||
def h(v: str) -> str:
|
||||
return hashlib.sha256(v.strip().strip('"').strip("'").encode()).hexdigest()[:16]
|
||||
|
||||
|
||||
def collect(src: dict[str, str]):
|
||||
"""(rows, values) where values[KEY] = {hash,...} for resolvability; nothing printed from it."""
|
||||
rows, values = [], {}
|
||||
for path, text in src.items():
|
||||
section = path
|
||||
for line in text.splitlines():
|
||||
m = HEAD.match(line) if path.endswith(".md") else None
|
||||
if m:
|
||||
section = safe(m.group(1), 90)
|
||||
continue
|
||||
m = ENV.match(line)
|
||||
if m:
|
||||
values.setdefault(m.group(1), set()).add(h(m.group(2)))
|
||||
rows.append((section, m.group(1), "file" if path.startswith("secrets/") else "env"))
|
||||
continue
|
||||
m = MD.match(line)
|
||||
if m:
|
||||
values.setdefault(m.group(1), set()).add(h(m.group(2)))
|
||||
rows.append((section, m.group(1), "md"))
|
||||
continue
|
||||
if path.endswith(".md"):
|
||||
mm = LABEL.search(line)
|
||||
if mm and not mm.group(1).startswith("http"):
|
||||
rows.append((section, safe(mm.group(1)), "label"))
|
||||
return rows, values
|
||||
|
||||
|
||||
def resolvable(label: str, kind: str, values) -> str:
|
||||
if kind not in ("env", "md", "file"):
|
||||
return "no"
|
||||
n = len(values.get(label, ()))
|
||||
return "yes" if n == 1 else "dup" if n > 1 else "no"
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
rx = re.compile(argv[0], re.I) if argv else None
|
||||
src = read_sources()
|
||||
if not src:
|
||||
print("lockbox-names: cannot read the lockbox")
|
||||
return 2
|
||||
rows, values = collect(src)
|
||||
seen, n = set(), 0
|
||||
for section, label, kind in rows:
|
||||
if rx and not (rx.search(section) or rx.search(label)):
|
||||
continue
|
||||
key = (section, label, kind)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
n += 1
|
||||
print(f"{section} | {label} | {kind} | {resolvable(label, kind, values)}")
|
||||
print(f"# {n} entr{'y' if n == 1 else 'ies'}; names only, values never printed")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
sys.exit(main())
|
||||
except re.error:
|
||||
print("lockbox-names: bad regex")
|
||||
sys.exit(2)
|
||||
except Exception as e: # never a traceback
|
||||
print(f"lockbox-names: error: {type(e).__name__}")
|
||||
sys.exit(2)
|
||||
@@ -54,7 +54,7 @@ REPOS = os.environ.get(
|
||||
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
||||
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
||||
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
||||
" windy-inbox windy-text windy-call windy-cell",
|
||||
" windy-inbox windy-text windy-call windy-cell windy-hand-site windy-calendar-site",
|
||||
).split()
|
||||
|
||||
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
||||
|
||||
209
scripts/runner_guard.py
Normal file
209
scripts/runner_guard.py
Normal file
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Runner guard: no workflow may let a STRANGER's code reach a self-hosted runner (Boss 10-01).
|
||||
|
||||
Our self-hosted GitHub runners run on Veron as `github-runner`, which is in the docker group
|
||||
(= root on Veron). Until ephemeral containerised runners exist (after launch), the cheap
|
||||
guard is to refuse the workflow shapes that hand a self-hosted runner to outsiders:
|
||||
|
||||
R1 pull_request_target : runs with secrets/write token in the BASE repo context
|
||||
R2 pull_request on self-hosted : fork PRs run their own code, unless the job is gated to
|
||||
same-repo heads (`if:` on head.repo.full_name == github.repository
|
||||
or head.repo.fork == false) or an `environment:`
|
||||
R3 issue_comment / workflow_run / issues / discussion* / pull_request_review* / fork / watch
|
||||
: anyone can fire these; never on self-hosted without an environment gate
|
||||
(push, tags, schedule, workflow_dispatch, repository_dispatch, workflow_call: writers only, fine)
|
||||
|
||||
A job counts as self-hosted when its runs-on names `self-hosted`, or is an expression we
|
||||
can't resolve (conservative). Findings carry file:line, never file content beyond that.
|
||||
|
||||
python3 scripts/runner_guard.py lint FILE... # local files
|
||||
python3 scripts/runner_guard.py report [--owners a,b] # default branch of every PUBLIC repo
|
||||
python3 scripts/runner_guard.py pr [--owners a,b] [--post] # open PRs on public repos: changed workflows
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
OWNERS = ["sneakyfree", "VERONTECH", "Windstorm-Institute", "Windstorm-Labs", "Public-Streamer"]
|
||||
CTX = "windy-git/runner-guard"
|
||||
OUTSIDE = {"issue_comment", "workflow_run", "issues", "discussion", "discussion_comment",
|
||||
"pull_request_review", "pull_request_review_comment", "fork", "watch"}
|
||||
SAME_REPO_GATES = ("head.repo.full_name == github.repository", "github.repository == github.event.pull_request.head.repo.full_name",
|
||||
"head.repo.fork == false", "!github.event.pull_request.head.repo.fork")
|
||||
|
||||
|
||||
def _node_map(node):
|
||||
"""{key: (value_node, line)} for a YAML mapping node."""
|
||||
if not isinstance(node, yaml.MappingNode):
|
||||
return {}
|
||||
return {k.value: (v, k.start_mark.line + 1) for k, v in node.value if isinstance(k, yaml.ScalarNode)}
|
||||
|
||||
|
||||
def _triggers(on_node) -> dict[str, int]:
|
||||
"""{event: line}."""
|
||||
if isinstance(on_node, yaml.ScalarNode):
|
||||
return {on_node.value: on_node.start_mark.line + 1}
|
||||
if isinstance(on_node, yaml.SequenceNode):
|
||||
return {n.value: n.start_mark.line + 1 for n in on_node.value if isinstance(n, yaml.ScalarNode)}
|
||||
return {k: line for k, (_v, line) in _node_map(on_node).items()}
|
||||
|
||||
|
||||
def _self_hosted(runs_on) -> bool:
|
||||
if runs_on is None:
|
||||
return False
|
||||
text = yaml.serialize(runs_on) if isinstance(runs_on, yaml.Node) else str(runs_on)
|
||||
return "self-hosted" in text or "${{" in text
|
||||
|
||||
|
||||
def lint_text(path: str, text: str) -> list[tuple[str, int, str, str]]:
|
||||
"""[(path, line, rule, message)]. Unparseable YAML is a finding (it cannot be reviewed)."""
|
||||
try:
|
||||
root = yaml.compose(text)
|
||||
except yaml.YAMLError as e:
|
||||
line = getattr(getattr(e, "problem_mark", None), "line", 0) + 1
|
||||
return [(path, line, "R0", "workflow YAML does not parse; cannot be checked")]
|
||||
top = _node_map(root)
|
||||
if "on" not in top or "jobs" not in top:
|
||||
return []
|
||||
trig = _triggers(top["on"][0])
|
||||
jobs = _node_map(top["jobs"][0])
|
||||
out = []
|
||||
if "pull_request_target" in trig:
|
||||
out.append((path, trig["pull_request_target"], "R1",
|
||||
"pull_request_target runs fork code with base-repo secrets; not allowed"))
|
||||
for name, (jnode, jline) in jobs.items():
|
||||
j = _node_map(jnode)
|
||||
ro = j.get("runs-on", (None, jline))
|
||||
if not _self_hosted(ro[0]):
|
||||
continue
|
||||
has_env = "environment" in j
|
||||
cond = j["if"][0].value if "if" in j and isinstance(j["if"][0], yaml.ScalarNode) else ""
|
||||
same_repo = any(g in cond.replace(" ", " ") for g in SAME_REPO_GATES)
|
||||
if "pull_request" in trig and not (has_env or same_repo):
|
||||
out.append((path, ro[1], "R2", f"job '{name}' runs fork PR code on a self-hosted runner "
|
||||
"(gate it: if: github.event.pull_request.head.repo.full_name == github.repository, or an environment)"))
|
||||
for ev in sorted(OUTSIDE & trig.keys()):
|
||||
if not has_env:
|
||||
out.append((path, trig[ev], "R3", f"'{ev}' can be fired by anyone and job '{name}' is self-hosted "
|
||||
"without an environment gate"))
|
||||
return out
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- GitHub side
|
||||
def gh(*args: str, check=True) -> str:
|
||||
r = subprocess.run(["gh", "api", *args], capture_output=True, text=True, timeout=60)
|
||||
if check and r.returncode != 0:
|
||||
raise RuntimeError(f"gh api {args[0]} failed")
|
||||
return r.stdout
|
||||
|
||||
|
||||
def public_repos(owners) -> list[tuple[str, str]]:
|
||||
out = []
|
||||
for o in owners:
|
||||
txt = gh(f"users/{o}/repos?per_page=100&type=owner", "--paginate",
|
||||
"--jq", '.[]|select(.private==false and .archived==false)|.full_name+" "+.default_branch', check=False)
|
||||
out += [tuple(row.split()) for row in txt.splitlines() if row.strip()]
|
||||
return out
|
||||
|
||||
|
||||
def workflows_at(full: str, ref: str) -> list[tuple[str, str]]:
|
||||
txt = gh(f"repos/{full}/contents/.github/workflows?ref={ref}", "--jq",
|
||||
'.[]|select(.type=="file")|.path', check=False)
|
||||
files = [p for p in txt.splitlines() if p.endswith((".yml", ".yaml"))]
|
||||
return [(p, file_at(full, p, ref)) for p in files]
|
||||
|
||||
|
||||
def file_at(full: str, path: str, ref: str) -> str:
|
||||
raw = gh(f"repos/{full}/contents/{path}?ref={ref}", "--jq", ".content", check=False).strip()
|
||||
return base64.b64decode(raw).decode("utf-8", "replace") if raw else ""
|
||||
|
||||
|
||||
def cmd_report(owners) -> int:
|
||||
hits = 0
|
||||
repos = public_repos(owners)
|
||||
for full, branch in repos:
|
||||
for path, text in workflows_at(full, branch):
|
||||
for p, line, rule, msg in lint_text(path, text):
|
||||
hits += 1
|
||||
print(f"{full}\t{p}:{line}\t{rule}\t{msg}")
|
||||
print(f"# runner-guard sweep: {hits} hit(s) in {len(repos)} public repos")
|
||||
return 1 if hits else 0
|
||||
|
||||
|
||||
STATE = os.environ.get("RUNNER_GUARD_STATE", "/var/lib/windy-git/runner-guard-posted.json")
|
||||
|
||||
|
||||
def cmd_pr(owners, post: bool) -> int:
|
||||
# Post each (repo, sha, state, description) ONCE: the sync runs every 5 min and GitHub caps
|
||||
# statuses per sha+context at 1000.
|
||||
try:
|
||||
with open(STATE) as fh:
|
||||
posted = set(json.load(fh))
|
||||
except (OSError, ValueError):
|
||||
posted = set()
|
||||
seen = set()
|
||||
for full, _branch in public_repos(owners):
|
||||
prs = gh(f"repos/{full}/pulls?state=open&per_page=50", "--jq",
|
||||
'.[]|(.number|tostring)+" "+.head.sha+" "+.head.repo.full_name', check=False)
|
||||
for line in prs.splitlines():
|
||||
num, sha, head_repo = line.split(" ", 2)
|
||||
files = gh(f"repos/{full}/pulls/{num}/files?per_page=100", "--jq",
|
||||
'.[]|select(.status!="removed")|.filename', check=False).split()
|
||||
wf = [f for f in files if f.startswith(".github/workflows/") and f.endswith((".yml", ".yaml"))]
|
||||
# a fork's own content is read from the head repo at the head sha
|
||||
src = head_repo if head_repo and head_repo != "null" else full
|
||||
found = [h for f in wf for h in lint_text(f, file_at(src, f, sha))]
|
||||
if found:
|
||||
p, ln, rule, msg = found[0]
|
||||
state, desc = "failure", f"BLOCKED: {rule} {p}:{ln}: {msg}"[:140]
|
||||
else:
|
||||
state, desc = "success", ("OK: no workflow changes" if not wf else
|
||||
"OK: no stranger-code path to a self-hosted runner")
|
||||
key = f"{full}@{sha}:{state}:{desc}"
|
||||
seen.add(key)
|
||||
if key in posted:
|
||||
continue
|
||||
print(f"{full}#{num}@{sha[:7]} {state} {desc}")
|
||||
if post:
|
||||
gh(f"repos/{full}/statuses/{sha}", "-f", f"state={state}", "-f", f"context={CTX}",
|
||||
"-f", f"description={desc}", check=False)
|
||||
posted.add(key)
|
||||
if post: # keep only keys for PRs still open, so the file never grows without bound
|
||||
os.makedirs(os.path.dirname(STATE), exist_ok=True)
|
||||
with open(STATE, "w") as fh:
|
||||
json.dump(sorted(posted & seen), fh)
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv=None) -> int:
|
||||
ap = argparse.ArgumentParser(prog="runner_guard")
|
||||
sub = ap.add_subparsers(dest="cmd", required=True)
|
||||
lint_p = sub.add_parser("lint")
|
||||
lint_p.add_argument("files", nargs="+")
|
||||
rep = sub.add_parser("report")
|
||||
rep.add_argument("--owners", default=",".join(OWNERS))
|
||||
prp = sub.add_parser("pr")
|
||||
prp.add_argument("--owners", default="sneakyfree") # self-hosted runners exist only there
|
||||
prp.add_argument("--post", action="store_true")
|
||||
a = ap.parse_args(argv)
|
||||
if a.cmd == "lint":
|
||||
hits = []
|
||||
for f in a.files:
|
||||
with open(f, errors="replace") as fh:
|
||||
hits += lint_text(f, fh.read())
|
||||
for p_, ln, rule, msg in hits:
|
||||
print(f"{p_}:{ln}\t{rule}\t{msg}")
|
||||
return 1 if hits else 0
|
||||
owners = a.owners.split(",")
|
||||
return cmd_report(owners) if a.cmd == "report" else cmd_pr(owners, a.post)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -38,7 +38,7 @@ FAILED=0
|
||||
|
||||
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
||||
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell}"
|
||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell windy-hand-site windy-calendar-site}"
|
||||
|
||||
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
||||
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
||||
@@ -94,6 +94,11 @@ if ! python3 "$(dirname "$0")/pr_status_bridge.py"; then
|
||||
log "FAILED pr status bridge"; FAILED=1
|
||||
fi
|
||||
|
||||
# Runner guard (Boss 10-01): PUBLIC sneakyfree repos have self-hosted GitHub runners on Veron.
|
||||
# A PR that changes a workflow so a stranger's code could reach one gets a red
|
||||
# windy-git/runner-guard status. Each status is posted once; never fails the sync.
|
||||
timeout -k 10 120 python3 "$(dirname "$0")/runner_guard.py" pr --post || log "runner-guard failed or timed out (non-fatal)"
|
||||
|
||||
# CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
|
||||
# Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
|
||||
timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)"
|
||||
|
||||
Reference in New Issue
Block a user