Compare commits
7 Commits
9566826ce9
...
veron-olla
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cd0400c371 | ||
|
|
4e7ab667ed | ||
|
|
f10db19316 | ||
|
|
3503894a1e | ||
|
|
fbab5c4669 | ||
|
|
1b552c0882 | ||
|
|
8ebc18c3bc |
@@ -229,3 +229,24 @@ def test_block_mode_never_blocks_grant_owned(monkeypatch):
|
||||
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
||||
lane = cg.Finding("a.py", 3, "provider host", "x")
|
||||
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
||||
|
||||
|
||||
def test_veron_ollama_warns_on_added_lines_and_never_blocks(monkeypatch):
|
||||
hits = cg.scan_line("app/llm.py", 'OLLAMA = "http://192.168.1.73:11434/api/generate"')
|
||||
assert [k for k, _ in hits] == ["veron ollama"]
|
||||
assert cg.scan_line("app/llm.py", 'port = 114345') == [] # not the port
|
||||
assert cg.scan_line("app/llm.py", "# was http://x:11434 (removed)") == [] # a comment is not a call
|
||||
f = cg.Finding("app/llm.py", 7, "veron ollama", ":11434")
|
||||
monkeypatch.setattr(cg, "MODE", "block")
|
||||
state, desc, _ = cg.status_for([f], whole_tree=False)
|
||||
assert state == "success" and desc.startswith("⚠ WARN: new Veron Ollama ref app/llm.py:7")
|
||||
assert "Windy Mind" in desc and len(desc) <= 140
|
||||
hard = cg.Finding("app/llm.py", 1, "provider host", "api.openai.com")
|
||||
assert cg.status_for([f, hard], whole_tree=False)[0] == "failure" # a real violation still blocks
|
||||
|
||||
|
||||
def test_ollama_in_added_pr_lines_only():
|
||||
diff = ("+++ b/svc/client.py\n@@ -0,0 +1,2 @@\n+import httpx\n"
|
||||
"+URL = 'http://veron:11434/api/chat'\n")
|
||||
got = cg.parse_added("some-repo", diff, [])
|
||||
assert [(f.kind, f.line) for f in got] == [("veron ollama", 2)]
|
||||
|
||||
@@ -146,3 +146,11 @@ def test_warn_kinds_do_not_block(monkeypatch):
|
||||
assert sg.status_for([f], True)[0] == "success"
|
||||
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
||||
assert sg.status_for([f], True)[0] == "failure"
|
||||
|
||||
|
||||
def test_pypi_token_shape_hash_only():
|
||||
tok = "pypi-AgE" + "Ab1_-" * 20 # synthetic
|
||||
got = ss.find(f"password = {tok}")
|
||||
assert [k for k, _ in got] == ["pypi token"] and got[0][1] == ss.h8(tok)
|
||||
assert tok not in repr(got)
|
||||
assert ss.find("pypi-AgE-too-short") == []
|
||||
|
||||
@@ -45,3 +45,8 @@ allow:
|
||||
- repo: windy-git
|
||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||
reason: "The guard's own pattern list and this file."
|
||||
|
||||
- repo: windy-mind
|
||||
paths: ["*"]
|
||||
matches: [':11434']
|
||||
reason: "Windy Mind IS the compute door (endpoint + key); it may call Ollama. Only the Ollama port is allowed here, any provider host/SDK in Mind still flags."
|
||||
|
||||
14
deploy/systemd/windygit-state-backup.service
Normal file
14
deploy/systemd/windygit-state-backup.service
Normal file
@@ -0,0 +1,14 @@
|
||||
[Unit]
|
||||
Description=Windy Git nightly STATE backup (Postgres + Gitea config + repos -> encrypted restic in R2)
|
||||
After=network-online.target docker.service
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
WorkingDirectory=/srv/windygit/src
|
||||
# R2 credentials come from the .env; the restic password from /etc/windygit/restic.pass
|
||||
# (root 600; the same value lives in the lockbox as RESTIC_WINDYGIT_PASSWORD).
|
||||
EnvironmentFile=/srv/windygit/src/.env
|
||||
ExecStart=/bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||
Nice=10
|
||||
IOSchedulingClass=idle
|
||||
TimeoutStartSec=3h
|
||||
@@ -0,0 +1,3 @@
|
||||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=/usr/local/bin/windy-job windygit-state-backup 26h --expect "ok — state backed up" --owner 13 -- /bin/bash /srv/windygit/src/scripts/backup_state.sh
|
||||
11
deploy/systemd/windygit-state-backup.timer
Normal file
11
deploy/systemd/windygit-state-backup.timer
Normal file
@@ -0,0 +1,11 @@
|
||||
[Unit]
|
||||
Description=Nightly Windy Git state backup
|
||||
|
||||
[Timer]
|
||||
# 03:07 local, clear of the git-bundle backup at 04:17.
|
||||
OnCalendar=*-*-* 03:07:00
|
||||
Persistent=true
|
||||
RandomizedDelaySec=300
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
@@ -32,7 +32,7 @@ services:
|
||||
|
||||
gitea:
|
||||
# G2.1 — PIN AN EXACT VERSION. Never `latest`. Record it in SUBSTRATE.md.
|
||||
image: docker.io/gitea/gitea:1.24.6
|
||||
image: docker.io/gitea/gitea:1.24.7
|
||||
environment:
|
||||
GITEA__database__DB_TYPE: postgres
|
||||
GITEA__database__HOST: db:5432
|
||||
|
||||
33
docs/RESTORE-DRILL.md
Normal file
33
docs/RESTORE-DRILL.md
Normal file
@@ -0,0 +1,33 @@
|
||||
# Restoring Windy Git from the encrypted state backup
|
||||
|
||||
What is backed up (`scripts/backup_state.sh`, restic repo `s3:…/windy-git-backups/restic`, tag `windygit-state`):
|
||||
both Postgres databases (`gitea`, `windygit`, custom-format dumps + globals), the whole Gitea data root
|
||||
(`/srv/windygit/git`: config, jwt, attachments, avatars, templates AND the bare repositories),
|
||||
`/srv/windygit/src/.env`, `deploy/runner/.env`, `/etc/cloudflared`, the windygit systemd drop-ins.
|
||||
NOT in it: the restic password itself (lockbox `RESTIC_WINDYGIT_PASSWORD`) and the R2 access key
|
||||
(scoped token `windy-git-r2-scoped`, lockbox). Never print either: use `lockbox-get KEY FILE`.
|
||||
|
||||
## Drill (any machine with restic + docker; proven on Veron 2026-10-01, counts identical)
|
||||
export RESTIC_PASSWORD_FILE=<0600 file from lockbox-get RESTIC_WINDYGIT_PASSWORD>
|
||||
export AWS_ACCESS_KEY_ID=… AWS_SECRET_ACCESS_KEY=… # from lockbox-get, into env, not echoed
|
||||
export RESTIC_REPOSITORY=s3:https://<R2 account>.r2.cloudflarestorage.com/windy-git-backups/restic
|
||||
restic snapshots --tag windygit-state
|
||||
restic restore latest --tag windygit-state --target /var/tmp/wg-drill
|
||||
docker run -d --name wg-drill-pg -e POSTGRES_PASSWORD=<random> -e POSTGRES_USER=drill postgres:16-alpine
|
||||
for db in gitea windygit; do
|
||||
docker exec wg-drill-pg psql -U drill -d postgres -c "create database $db"
|
||||
docker exec -i wg-drill-pg pg_restore -U drill -d $db --no-owner --no-privileges \
|
||||
< /var/tmp/wg-drill/var/backups/windygit-state/$db.dump
|
||||
done
|
||||
# compare row counts with live (or with the last known): repository, issue, pull_request, "user",
|
||||
# external_login_user, access_token, action_run, action_run_job
|
||||
docker rm -f wg-drill-pg; rm -rf /var/tmp/wg-drill
|
||||
|
||||
## Real disaster (Veron lost)
|
||||
1. New Linux host with Docker, a Cloudflare tunnel connector, the repo (`git clone` from GitHub: windy-git).
|
||||
2. `restic restore latest --tag windygit-state --target /` (puts /srv/windygit/git, the .env files, /etc/cloudflared back).
|
||||
3. `docker compose -p windy-git up -d db`, then pg_restore both dumps into it (as above, into the real db names/owner from `.env`).
|
||||
4. `docker compose -p windy-git up -d` + `deploy/runner` runners; re-register runners if the token changed.
|
||||
5. Verify: `/api/healthz`, Windy SSO login, `git ls-remote`, one CI run. GitHub is still the source of truth for code,
|
||||
so repo content can also be re-synced from there; the database is what only this backup holds.
|
||||
Retention: 14 daily / 8 weekly / 6 monthly (prune on Sundays). Integrity: every run does `restic check --read-data-subset=2%`.
|
||||
@@ -22,7 +22,16 @@ GIT_ROOT="${GIT_DATA_ROOT:-/srv/windygit/git}"
|
||||
umask 077
|
||||
mkdir -p "$STAGE"; chmod 700 "$STAGE"; rm -f "$STAGE"/*.dump "$STAGE"/globals.sql
|
||||
|
||||
restic cat config >/dev/null 2>&1 || { log "initialising restic repo"; restic init >/dev/null; }
|
||||
# systemd gives units no $HOME, and restic wants a cache dir: pin one.
|
||||
export RESTIC_CACHE_DIR="${RESTIC_CACHE_DIR:-/var/cache/windygit-restic}"; mkdir -p "$RESTIC_CACHE_DIR"
|
||||
if ! err=$(restic cat config 2>&1 >/dev/null); then
|
||||
# only a MISSING repo may be initialised; any other error (auth, network, wrong password) must stop here
|
||||
if grep -qiE "does not exist|is there a repository|unable to open config file" <<<"$err"; then
|
||||
log "initialising restic repo"; restic init >/dev/null
|
||||
else
|
||||
log "FATAL: restic cannot open the repository: $(head -c 300 <<<"$err" | tr '\n' ' ')"; exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
PGU=$(timeout 30 docker exec "$DB" printenv POSTGRES_USER)
|
||||
[[ -n "$PGU" ]] || { log "FATAL: no POSTGRES_USER in $DB"; exit 1; }
|
||||
|
||||
@@ -63,6 +63,9 @@ JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/gen
|
||||
|
||||
RULES: list[tuple[str, re.Pattern]] = [
|
||||
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||
# Grant via Boss 10-01: compute = Windy Mind. A NEW reference to an Ollama port (Veron's :11434) is a
|
||||
# direct call around Mind's metering/caps. WARN-only, never red, and only for lines a PR ADDS.
|
||||
("veron ollama", re.compile(r"(?::|%3[aA])11434(?![0-9])")),
|
||||
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||
@@ -70,6 +73,9 @@ RULES: list[tuple[str, re.Pattern]] = [
|
||||
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||
]
|
||||
# Kinds that never block (even in MODE=block) and are only judged on ADDED lines, never the baseline tree.
|
||||
WARN_ONLY_KINDS = {"veron ollama"}
|
||||
OLLAMA_MSG = "compute = Windy Mind (endpoint + key); do not call Veron's Ollama directly"
|
||||
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||
|
||||
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||
@@ -253,7 +259,8 @@ def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> li
|
||||
allow = load_allow()
|
||||
fp = _fingerprint(allow)
|
||||
if is_default_head:
|
||||
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
||||
return cached_scan(f"tree:{repo}:{sha}:{fp}",
|
||||
lambda: [f for f in scan_tree(repo, bare, sha, allow) if f.kind not in WARN_ONLY_KINDS])
|
||||
return cached_scan(
|
||||
f"pr:{repo}:{sha}:{fp}",
|
||||
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||
@@ -268,6 +275,11 @@ def status_for(findings: list[Finding], whole_tree: bool,
|
||||
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
||||
09-23: his desktop work is never blocked by us)."""
|
||||
scope = "in tree" if whole_tree else "added"
|
||||
soft = [f for f in findings if f.kind in WARN_ONLY_KINDS]
|
||||
findings = [f for f in findings if f.kind not in WARN_ONLY_KINDS]
|
||||
if not findings and not grant and soft:
|
||||
f = soft[0]
|
||||
return "success", f"⚠ WARN: new Veron Ollama ref {f.path}:{f.line}. {OLLAMA_MSG}"[:140], f
|
||||
if not findings and grant:
|
||||
g, n = grant[0], len(grant)
|
||||
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
||||
|
||||
23
scripts/drill_cross_host.sh
Executable file
23
scripts/drill_cross_host.sh
Executable file
@@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
# Cross-host restore drill on Windy 0: ONLY lockbox + R2, nothing from Veron. Values never printed.
|
||||
# Usage: bash drill_cross_host.sh (needs lockbox keys RESTIC_WINDYGIT_PASSWORD, WINDYGIT_R2_ACCESS_KEY_ID, WINDYGIT_R2_SECRET_ACCESS_KEY, WINDYGIT_R2_ENDPOINT)
|
||||
set -euo pipefail
|
||||
umask 077; W=$(mktemp -d ~/.cache/wg-xdrill.XXXXXX)
|
||||
trap 'docker rm -f wg-xdrill-pg >/dev/null 2>&1 || true; rm -rf "$W"' EXIT
|
||||
lockbox-get RESTIC_WINDYGIT_PASSWORD "$W/pw" >/dev/null
|
||||
lockbox-get WINDYGIT_R2_ACCESS_KEY_ID "$W/ak" >/dev/null; lockbox-get WINDYGIT_R2_SECRET_ACCESS_KEY "$W/sk" >/dev/null; lockbox-get WINDYGIT_R2_ENDPOINT "$W/ep" >/dev/null
|
||||
export RESTIC_PASSWORD_FILE="$W/pw" AWS_ACCESS_KEY_ID="$(cat "$W/ak")" AWS_SECRET_ACCESS_KEY="$(cat "$W/sk")"
|
||||
export RESTIC_REPOSITORY="s3:$(cat "$W/ep")/windy-git-backups/restic"
|
||||
restic snapshots --tag windygit-state --compact | tail -3
|
||||
restic restore latest --tag windygit-state --target "$W/r" --include /var/backups/windygit-state --include /srv/windygit/git/gitea/conf --quiet
|
||||
ls -l "$W/r/var/backups/windygit-state" | awk 'NR>1{print $5, $NF}'
|
||||
docker run -d --name wg-xdrill-pg -e POSTGRES_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_hex(12))')" -e POSTGRES_USER=drill postgres:16-alpine >/dev/null
|
||||
for i in $(seq 1 30); do docker exec wg-xdrill-pg pg_isready -U drill >/dev/null 2>&1 && break; sleep 2; done
|
||||
for db in gitea windygit; do
|
||||
docker exec wg-xdrill-pg psql -U drill -d postgres -qc "create database $db"
|
||||
docker exec -i wg-xdrill-pg pg_restore -U drill -d $db --no-owner --no-privileges < "$W/r/var/backups/windygit-state/$db.dump" 2>&1 | grep -v "already exists" | head -2 || true
|
||||
done
|
||||
for t in repository issue pull_request '"user"' external_login_user action_run action_run_job; do
|
||||
echo "$t restored=$(docker exec wg-xdrill-pg psql -U drill -d gitea -Atc "select count(*) from $t")"
|
||||
done
|
||||
echo "cross-host drill OK (cleaned up)"
|
||||
@@ -23,13 +23,14 @@ PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
||||
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
||||
("32-hex secret assignment", re.compile(
|
||||
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
||||
("pypi token", re.compile(r"\bpypi-AgE[A-Za-z0-9_-]{50,}")),
|
||||
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
||||
]
|
||||
|
||||
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
||||
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
||||
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}")
|
||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}|pypi-AgE")
|
||||
|
||||
|
||||
def h8(value: str | bytes) -> str:
|
||||
|
||||
Reference in New Issue
Block a user