Compare commits
4 Commits
fe3bce39ff
...
ci-inputs-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2daca61ef | ||
|
|
1bc55eaec9 | ||
|
|
fdb0f5989e | ||
|
|
9be2952ff8 |
184
api/tests/test_compute_guard.py
Normal file
184
api/tests/test_compute_guard.py
Normal file
@@ -0,0 +1,184 @@
|
||||
"""Compute guard: Windy Mind is the only door to AI compute (warn-only today)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
_spec = importlib.util.spec_from_file_location("compute_guard", ROOT / "scripts" / "compute_guard.py")
|
||||
cg = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["compute_guard"] = cg
|
||||
_spec.loader.exec_module(cg)
|
||||
|
||||
ALLOW = cg.load_allow(ROOT / "ci" / "compute-guard-allow.yml")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, text, kind",
|
||||
[
|
||||
# audit #1 (windy-search, closed) and #2 (windy-chat, live): the shapes they had
|
||||
("service/app/anthropic_client.py", 'URL = "https://api.anthropic.com/v1/messages"', "provider host"),
|
||||
("service/app/config.py", 'token = os.environ["ANTHROPIC_OAUTH_TOKEN"]', "provider key"),
|
||||
("services/agent-roster/lib/llm.js", "const url = 'https://api.groq.com/openai/v1/chat/completions'", "provider host"),
|
||||
("docker-compose.yml", " GROQ_API_KEY: ${GROQ_API_KEY}", "provider key"),
|
||||
# audit #3/#4 (windy-pro account-server)
|
||||
("account-server/src/routes/transcription.ts", "const r = await fetch('https://api.openai.com/v1/audio/transcriptions'", "provider host"),
|
||||
("account-server/src/config.ts", "openaiKey: process.env.OPENAI_API_KEY,", "provider key"),
|
||||
# SDKs and deps
|
||||
("app/llm.py", "from anthropic import Anthropic", "provider SDK"),
|
||||
("app/llm.py", "import openai", "provider SDK"),
|
||||
("app/llm.py", "import google.generativeai as genai", "provider SDK"),
|
||||
("src/ai.ts", 'import Anthropic from "@anthropic-ai/sdk";', "provider SDK"),
|
||||
("src/ai.js", "const Groq = require('groq-sdk')", "provider SDK"),
|
||||
("package.json", ' "openai": "^4.52.0",', "provider SDK dep"),
|
||||
("requirements.txt", "anthropic>=0.40", "provider SDK dep"),
|
||||
("pyproject.toml", ' "google-generativeai>=0.8",', "provider SDK dep"),
|
||||
],
|
||||
)
|
||||
def test_audit_shapes_are_flagged(path, text, kind):
|
||||
assert kind in [k for k, _ in cg.scan_line(path, text)]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path, text",
|
||||
[
|
||||
("app/mind.py", 'MIND = "https://mind.windyword.ai/v1/chat/completions"'), # the door itself
|
||||
("app/models.py", "openai_compatible = True # Mind speaks the OpenAI wire format"),
|
||||
("app/x.py", "from app.openai_shim import x"), # a local module, not the SDK
|
||||
("package.json", ' "openai-types-lite": "1.0.0",'), # a different package
|
||||
("README.txt", "set OPENAI_API_KEY"), # scanned-by-rule, excluded by SKIP separately
|
||||
],
|
||||
)
|
||||
def test_near_misses_are_not_flagged(path, text):
|
||||
if cg.SKIP.search(path):
|
||||
return
|
||||
assert cg.scan_line(path, text) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
["tests/test_llm.py", "api/tests/x.py", "src/ai.test.ts", "web/foo.spec.js", "docs/setup.md",
|
||||
"README.md", "package-lock.json", "uv.lock", "node_modules/openai/index.js", ".github/workflows/ci.yml",
|
||||
"conftest.py", "app/llm_test.py"],
|
||||
)
|
||||
def test_tests_docs_lockfiles_vendored_ci_are_never_scanned(path):
|
||||
assert cg.SKIP.search(path)
|
||||
|
||||
|
||||
def test_allow_list_needs_a_reason_per_entry(tmp_path):
|
||||
bad = tmp_path / "a.yml"
|
||||
bad.write_text("allow:\n - repo: x\n paths: ['*']\n")
|
||||
with pytest.raises(ValueError):
|
||||
cg.load_allow(bad)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"repo, path, ok",
|
||||
[
|
||||
("windy-mind", "app/providers/anthropic.py", True),
|
||||
("windy-agent", "agent/providers.py", True),
|
||||
("windy-code", "extensions/windy-ai/src/aiProvider.ts", True),
|
||||
("windy-code", "web/server/llm.ts", False), # BYOK is the extension only
|
||||
("windy-connect", "backend/src/writers/claude_code.py", True),
|
||||
("windy-chat", "services/agent-roster/lib/llm.js", False), # audit #2: must be flagged
|
||||
("windy-pro", "account-server/src/routes/translations.ts", False),
|
||||
],
|
||||
)
|
||||
def test_allow_list_entries(repo, path, ok):
|
||||
assert cg.allowed(repo, path, ALLOW) is ok
|
||||
|
||||
|
||||
DIFF = """diff --git a/app/llm.py b/app/llm.py
|
||||
--- a/app/llm.py
|
||||
+++ b/app/llm.py
|
||||
@@ -10,0 +11,2 @@
|
||||
+import anthropic
|
||||
+client = anthropic.Anthropic()
|
||||
diff --git a/tests/test_llm.py b/tests/test_llm.py
|
||||
--- /dev/null
|
||||
+++ b/tests/test_llm.py
|
||||
@@ -0,0 +1 @@
|
||||
+import anthropic
|
||||
@@ -40 +42 @@
|
||||
-x = 1
|
||||
+x = 2
|
||||
"""
|
||||
|
||||
|
||||
def test_only_added_non_test_lines_are_findings():
|
||||
fs = cg.parse_added("windy-chat", DIFF, ALLOW)
|
||||
assert [(f.path, f.line, f.kind) for f in fs] == [("app/llm.py", 11, "provider SDK")]
|
||||
|
||||
|
||||
def _repo(tmp_path, files: dict[str, str]) -> tuple[Path, str]:
|
||||
work = tmp_path / "w"
|
||||
work.mkdir()
|
||||
run = lambda *a: subprocess.run(["git", *a], cwd=work, check=True, capture_output=True) # noqa: E731
|
||||
run("init", "-q", "-b", "main")
|
||||
for p, text in files.items():
|
||||
(work / p).parent.mkdir(parents=True, exist_ok=True)
|
||||
(work / p).write_text(text)
|
||||
run("add", "-A")
|
||||
run("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "x")
|
||||
bare = tmp_path / "r.git"
|
||||
subprocess.run(["git", "clone", "-q", "--bare", str(work), str(bare)], check=True)
|
||||
sha = subprocess.run(["git", "--git-dir", str(bare), "rev-parse", "main"],
|
||||
capture_output=True, text=True, check=True).stdout.strip()
|
||||
return bare, sha
|
||||
|
||||
|
||||
def test_tree_scan_on_a_real_git_repo(tmp_path):
|
||||
bare, sha = _repo(tmp_path, {
|
||||
"app/llm.py": "import os\nKEY = os.environ['OPENAI_API_KEY']\n",
|
||||
"app/ok.py": "MIND = 'https://mind.windyword.ai'\n",
|
||||
"tests/test_llm.py": "import anthropic\n",
|
||||
"docs/x.md": "api.anthropic.com\n",
|
||||
})
|
||||
fs = cg.scan_tree("windy-chat", bare, sha, ALLOW)
|
||||
assert [(f.path, f.line, f.kind) for f in fs] == [("app/llm.py", 2, "provider key")]
|
||||
|
||||
|
||||
def test_warn_mode_never_turns_red(monkeypatch):
|
||||
monkeypatch.setattr(cg, "MODE", "warn")
|
||||
state, desc, f = cg.status_for([cg.Finding("a.py", 3, "provider host", "api.openai.com")], whole_tree=False)
|
||||
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 direct AI-provider use added")
|
||||
assert "a.py:3" in desc and f.path == "a.py"
|
||||
|
||||
|
||||
def test_block_mode_fails(monkeypatch):
|
||||
monkeypatch.setattr(cg, "MODE", "block")
|
||||
state, desc, _ = cg.status_for([cg.Finding("a.py", 3, "provider host", "x")], whole_tree=True)
|
||||
assert state == "failure" and desc.startswith("BLOCKED")
|
||||
|
||||
|
||||
def test_clean_is_ok():
|
||||
assert cg.status_for([], whole_tree=True)[:2] == (
|
||||
"success", "OK: no direct AI-provider use in tree (Windy Mind is the only door)")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
" # The ANTHROPIC_OAUTH_TOKEN setting was removed on 2026-09-23 ON PURPOSE", # windy-search
|
||||
"# ANTHROPIC_API_KEY=",
|
||||
" // fallback used to call https://api.groq.com directly",
|
||||
" * @see https://api.openai.com/v1/audio",
|
||||
"<!-- api.anthropic.com -->",
|
||||
],
|
||||
)
|
||||
def test_comments_are_not_calls(text):
|
||||
assert cg.scan_line("service/app/config.py", text) == []
|
||||
|
||||
|
||||
def test_code_with_a_trailing_comment_still_counts():
|
||||
assert cg.scan_line("a.js", "fetch('https://api.openai.com/v1') // TODO move to Mind")
|
||||
|
||||
|
||||
def test_windy_pro_desktop_is_byok_but_the_account_server_is_not():
|
||||
assert cg.allowed("windy-pro", "src/client/desktop/main.js", ALLOW)
|
||||
assert not cg.allowed("windy-pro", "account-server/src/routes/translations.ts", ALLOW)
|
||||
@@ -425,9 +425,28 @@ def test_i05_jobs_get_a_network_per_job_not_a_shared_bridge():
|
||||
|
||||
|
||||
def test_i05_jobs_cannot_bind_mount_from_the_daemon_host():
|
||||
cfg = (ROOT / "deploy" / "runner" / "config.yaml").read_text()
|
||||
assert "valid_volumes: []" in cfg
|
||||
assert 'docker_host: "-"' in cfg
|
||||
"""Narrowed 2026-09-23 (orchestrator-approved): a job may bind-mount EXACTLY
|
||||
one daemon path, windy-pro's non-secret build inputs, and only because dind
|
||||
itself has that path READ-ONLY. Anything more (a second path, a writable
|
||||
one, a glob) reopens the host to CI code. Still no docker socket for jobs."""
|
||||
import re
|
||||
|
||||
import yaml
|
||||
|
||||
rd = ROOT / "deploy" / "runner"
|
||||
cfg = yaml.safe_load((rd / "config.yaml").read_text())
|
||||
allowed = cfg["container"]["valid_volumes"]
|
||||
assert allowed in ([], ["/ci-inputs/windy-pro"]), f"I-5: jobs may mount nothing else: {allowed}"
|
||||
assert cfg["container"]["docker_host"] == "-"
|
||||
if allowed:
|
||||
compose = yaml.safe_load((rd / "docker-compose.yml").read_text())
|
||||
binds = [v for v in compose["services"]["dind"]["volumes"] if v.startswith("/")]
|
||||
assert binds == ["/home/user1-gpu/ci-inputs/windy-pro:/ci-inputs/windy-pro:ro"], (
|
||||
f"I-5: dind's only host bind must be the ci-inputs path, READ-ONLY: {binds}")
|
||||
for name, svc in compose["services"].items():
|
||||
if name != "dind":
|
||||
for v in svc.get("volumes") or []:
|
||||
assert not re.match(r"^/home/user1-gpu/ci-inputs", v), f"I-5: {name} mounts ci-inputs"
|
||||
|
||||
|
||||
def test_i05_no_ci_container_can_reach_the_forge_network():
|
||||
|
||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
@@ -340,3 +341,44 @@ def test_lookup_failure_is_non_fatal(monkeypatch):
|
||||
|
||||
monkeypatch.setattr(bridge.subprocess, "run", boom)
|
||||
assert bridge.queued_jobs("windy-chat", SHA) == []
|
||||
|
||||
|
||||
class _Guard:
|
||||
def __init__(self, findings):
|
||||
self.findings = findings
|
||||
|
||||
def check(self, repo, sha, default_branch, is_default_head):
|
||||
return self.findings
|
||||
|
||||
@staticmethod
|
||||
def status_for(findings, whole_tree):
|
||||
if not findings:
|
||||
return "success", "OK: clean", None
|
||||
return "success", f"WARN {len(findings)}", findings[0]
|
||||
|
||||
|
||||
class _F:
|
||||
path, line = "app/llm.py", 7
|
||||
|
||||
|
||||
def test_guard_posts_warn_with_a_link_to_the_first_finding(fake, monkeypatch):
|
||||
f = fake()
|
||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||
assert [(p["context"], p["state"], p["description"]) for p in f.posted] == [
|
||||
("windy-git/compute-guard", "success", "WARN 1")]
|
||||
assert f.posted[0]["target_url"].endswith(f"/src/commit/{SHA}/app/llm.py#L7")
|
||||
|
||||
|
||||
def test_guard_same_status_is_not_reposted(fake, monkeypatch):
|
||||
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "success", "description": "WARN 1"}])
|
||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||
assert f.posted == []
|
||||
|
||||
|
||||
def test_guard_that_cannot_run_posts_nothing(fake, monkeypatch):
|
||||
f = fake()
|
||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard(None))
|
||||
bridge.post_compute_guard("windy-chat", SHA, "main", True)
|
||||
assert f.posted == []
|
||||
|
||||
40
ci/compute-guard-allow.yml
Normal file
40
ci/compute-guard-allow.yml
Normal file
@@ -0,0 +1,40 @@
|
||||
# Compute guard allow-list: code that MAY talk to an AI provider directly.
|
||||
# Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23). Every entry
|
||||
# here is an exception to that rule and MUST say why. Paths are fnmatch globs
|
||||
# relative to the repo root. Owner of this file: Windy Git lane (13); changes
|
||||
# go through the orchestrator. Source of the first entries: COMPUTE_BYPASS_AUDIT.md.
|
||||
allow:
|
||||
- repo: windy-mind
|
||||
paths: ["*"]
|
||||
reason: "Windy Mind IS the door: provider clients belong here by definition."
|
||||
|
||||
- repo: windy-agent
|
||||
paths: ["*"]
|
||||
reason: >-
|
||||
User BYOK: self-hosted agents call providers on the USER's own keys.
|
||||
Mind stays opt-in there, or every self-hosted user's inference lands on
|
||||
Grant's bill (no-cloud-cost-liability rule; audit #7).
|
||||
|
||||
- repo: windy-code
|
||||
paths: ["extensions/windy-ai/*"]
|
||||
reason: "User BYOK AI extension: the user's own provider keys; Mind is one opt-in provider (audit #8)."
|
||||
|
||||
- repo: windy-connect
|
||||
paths: ["*writers/*"]
|
||||
reason: "Writes client configs that NAME the user's own provider env vars; makes no provider calls (audit #11)."
|
||||
|
||||
- repo: windy-pro
|
||||
paths: ["src/client/desktop/*"]
|
||||
reason: >-
|
||||
User BYOK desktop client: cloud STT/translate keys come from what the USER
|
||||
enters (renderer localStorage -> electron-store; env var only for dev), and
|
||||
the CSP line allows exactly those user-keyed hosts (audit #10). The
|
||||
account-server is NOT covered: server-side calls go through Mind.
|
||||
|
||||
- repo: windy-pro
|
||||
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
||||
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
||||
|
||||
- repo: windy-git
|
||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||
reason: "The guard's own pattern list and this file."
|
||||
@@ -54,6 +54,9 @@ container:
|
||||
privileged: false
|
||||
options:
|
||||
workdir_parent: /workspace
|
||||
valid_volumes: [] # a job cannot bind-mount anything from the daemon host
|
||||
# A job may bind-mount exactly ONE daemon path: the read-only windy-pro build
|
||||
# inputs (mounted :ro into dind itself). Per-runner, not per-repo (act_runner
|
||||
# limit): any windyadmin repo could mount it; it is non-secret and read-only.
|
||||
valid_volumes: ["/ci-inputs/windy-pro"]
|
||||
docker_host: "-" # do NOT expose the runner's own docker socket to jobs
|
||||
force_pull: false
|
||||
|
||||
15
deploy/runner/docker-compose.privileged.yml
Normal file
15
deploy/runner/docker-compose.privileged.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
# ROLLBACK ONLY: the pre-Sysbox dind (privileged: true), kept one command away.
|
||||
# Use it if CI breaks under Sysbox:
|
||||
#
|
||||
# cd /srv/windygit/src/deploy/runner
|
||||
# sudo docker compose -f docker-compose.yml -f docker-compose.privileged.yml up -d dind
|
||||
#
|
||||
# (then restart the runners while idle). Compose merges `volumes` by container
|
||||
# path, so this puts back the old `dind-storage` volume with its image cache.
|
||||
# Going forward again: the same command without the second -f.
|
||||
services:
|
||||
dind:
|
||||
runtime: runc
|
||||
privileged: true
|
||||
volumes:
|
||||
- dind-storage:/var/lib/docker
|
||||
@@ -26,8 +26,12 @@
|
||||
# * `dind` and every job container it spawns are UNTRUSTED. They are on a
|
||||
# private network with no access to the forge, its database, or its .env.
|
||||
#
|
||||
# dind itself is privileged — that is the cost, and it is the reason a job
|
||||
# escape lands in a disposable daemon rather than on Grant's workstation.
|
||||
# dind is NOT privileged (2026-09-23): it runs under the Sysbox runtime
|
||||
# (sysbox-ce on Veron, `runtime: sysbox-runc`), a user-namespaced system
|
||||
# container whose root is an unprivileged host uid. A job that escapes its own
|
||||
# container lands in dind as a nobody on the host, not as root on Grant's
|
||||
# workstation. Before Sysbox, dind was `privileged: true`; that config is kept
|
||||
# as docker-compose.privileged.yml (ROLLBACK ONLY, one command, see that file).
|
||||
#
|
||||
# ⚠️ Do NOT "simplify" this by mounting the host docker socket.
|
||||
|
||||
@@ -36,13 +40,20 @@ name: windy-git-runner
|
||||
services:
|
||||
dind:
|
||||
image: docker.io/library/docker:27-dind
|
||||
privileged: true
|
||||
runtime: sysbox-runc # NOT privileged: see the I-5 note above
|
||||
environment:
|
||||
DOCKER_TLS_CERTDIR: "" # plain TCP on an isolated network, no host route
|
||||
command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--tls=false"]
|
||||
networks: [jobs]
|
||||
volumes:
|
||||
- dind-storage:/var/lib/docker
|
||||
# A fresh volume: Sysbox shifts ownership to its own uid range. The old
|
||||
# `dind-storage` is kept untouched for the privileged rollback.
|
||||
- dind-storage-sysbox:/var/lib/docker
|
||||
# READ-ONLY, non-secret build inputs for windy-pro's desktop jobs (models,
|
||||
# linux-x64 portable bundle, enter-monitor build), copied from the frozen
|
||||
# release clone by deploy/runner/refresh-ci-inputs.sh. Jobs may mount ONLY
|
||||
# this path (config.yaml valid_volumes). Orchestrator-approved 09-23.
|
||||
- /home/user1-gpu/ci-inputs/windy-pro:/ci-inputs/windy-pro:ro
|
||||
# G1.5 — bounded so a fork-bomb workflow cannot starve Grant's interactive
|
||||
# session. Veron 1 is his workstation, not a dedicated build box.
|
||||
cpus: 12.0 # 12 of 24 cores
|
||||
@@ -159,6 +170,7 @@ networks:
|
||||
|
||||
volumes:
|
||||
dind-storage:
|
||||
dind-storage-sysbox:
|
||||
runner-data:
|
||||
runner-data-2:
|
||||
runner-data-3:
|
||||
|
||||
21
deploy/runner/refresh-ci-inputs.sh
Executable file
21
deploy/runner/refresh-ci-inputs.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
# Refresh the READ-ONLY CI input cache for windy-pro desktop jobs from the FROZEN
|
||||
# release clone on Veron. Reads ~/windy-pro-release only; never writes to it.
|
||||
# Run when the release lane says engines / wheels / the portable bundle changed.
|
||||
# Linux inputs only: 3 models + requirements-bundle.txt, bundled-portable/linux-x64,
|
||||
# native/enter-monitor/build. Result is chmod a-w and mounted :ro into dind.
|
||||
set -euo pipefail
|
||||
SRC=/home/user1-gpu/windy-pro-release
|
||||
DST=/home/user1-gpu/ci-inputs/windy-pro
|
||||
models=$(ls "$SRC/extraResources/model" | grep -E '^windy-(nano|lite|core)-ct2$')
|
||||
[ "$(wc -w <<<"$models")" = 3 ] || { echo "expected 3 models, got: $models"; exit 1; }
|
||||
mkdir -p "$DST/extraResources/model" "$DST/bundled-portable" "$DST/native-enter-monitor-build"
|
||||
chmod -R u+w "$DST"
|
||||
R="ionice -c3 nice -n 19 rsync -a --delete"
|
||||
for m in $models; do $R "$SRC/extraResources/model/$m/" "$DST/extraResources/model/$m/"; done
|
||||
$R "$SRC/extraResources/requirements-bundle.txt" "$DST/extraResources/requirements-bundle.txt"
|
||||
$R "$SRC/bundled-portable/linux-x64/" "$DST/bundled-portable/linux-x64/"
|
||||
$R "$SRC/native/enter-monitor/build/" "$DST/native-enter-monitor-build/"
|
||||
date -u +%FT%TZ > "$DST/.refreshed-from-windy-pro-release"
|
||||
chmod -R a-w "$DST"
|
||||
du -sh --apparent-size "$DST"
|
||||
270
scripts/compute_guard.py
Normal file
270
scripts/compute_guard.py
Normal file
@@ -0,0 +1,270 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Compute guard: Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23).
|
||||
|
||||
Flags code that talks to an AI provider directly instead of through Windy Mind:
|
||||
a provider API host, a provider SDK import or dependency, or a raw provider key
|
||||
name. Direct calls skip Mind's metering, caps and live-model routing, and they
|
||||
spend whichever key happens to be lying around (the audit found Grant's personal
|
||||
Max OAuth token inside a platform container).
|
||||
|
||||
WARN-ONLY for now: the bridge posts `windy-git/compute-guard` as success with a
|
||||
"⚠ WARN" description, so nothing turns red. `COMPUTE_GUARD_MODE=block` flips
|
||||
findings to failure once the repos are clean (orchestrator's call).
|
||||
|
||||
- PR heads: only lines the PR ADDS (vs its merge-base with the default branch).
|
||||
- Default-branch head: the whole tree (the baseline, and what `report` prints).
|
||||
|
||||
Exceptions live in ONE file, ci/compute-guard-allow.yml, each with a reason.
|
||||
Tests, docs, lockfiles, vendored code and CI config are never scanned.
|
||||
Reads the sync's bare GitHub clones on Veron (no docker exec: IO-stall lesson).
|
||||
|
||||
python3 scripts/compute_guard.py report [repo ...] # whole-tree findings on each default branch
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ALLOW_FILE = Path(os.environ.get("COMPUTE_GUARD_ALLOW", ROOT / "ci" / "compute-guard-allow.yml"))
|
||||
WORK = Path(os.environ.get("SYNC_WORK", "/srv/windygit/sync"))
|
||||
CACHE = Path(os.environ.get("COMPUTE_GUARD_CACHE", "/var/lib/windy-git/compute-guard-cache.json"))
|
||||
MODE = os.environ.get("COMPUTE_GUARD_MODE", "warn") # warn | block
|
||||
|
||||
HOSTS = [
|
||||
"api.anthropic.com", "api.openai.com", "api.groq.com",
|
||||
"generativelanguage.googleapis.com", "api.mistral.ai", "api.perplexity.ai",
|
||||
"openrouter.ai", "api.together.xyz", "api.together.ai", "api.cerebras.ai",
|
||||
"api.sambanova.ai", "api.deepseek.com", "api.x.ai", "api.cohere.ai",
|
||||
"api.cohere.com", "api.fireworks.ai", "api.replicate.com",
|
||||
"api-inference.huggingface.co",
|
||||
]
|
||||
KEYS = [
|
||||
"ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN",
|
||||
"OPENAI_API_KEY", "GROQ_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY",
|
||||
"GOOGLE_AI_API_KEY", "MISTRAL_API_KEY", "PERPLEXITY_API_KEY", "PPLX_API_KEY",
|
||||
"OPENROUTER_API_KEY", "TOGETHER_API_KEY", "CEREBRAS_API_KEY", "SAMBANOVA_API_KEY",
|
||||
"DEEPSEEK_API_KEY", "XAI_API_KEY", "COHERE_API_KEY", "FIREWORKS_API_KEY",
|
||||
"REPLICATE_API_TOKEN",
|
||||
]
|
||||
PY_SDKS = r"anthropic|openai|groq|mistralai|cohere|google\.generativeai|google\.genai|together|cerebras|litellm"
|
||||
JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/genai|@mistralai/mistralai"
|
||||
r"|cohere-ai|together-ai|@ai-sdk/(?:anthropic|openai|groq|google|mistral)")
|
||||
|
||||
RULES: list[tuple[str, re.Pattern]] = [
|
||||
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||
# dependency manifests: package.json keys, requirements / pyproject lines
|
||||
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||
]
|
||||
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||
|
||||
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||
SKIP = re.compile(
|
||||
r"(^|/)(tests?|__tests__|spec|docs?|node_modules|vendor|dist|build|\.github|\.gitea)/"
|
||||
r"|(^|/)(test_[^/]*|[^/]*_test\.py|conftest\.py|[^/]*\.(test|spec)\.[cm]?[jt]sx?)$"
|
||||
r"|\.(md|mdx|rst|txt|lock|snap|svg|png|jpg|pdf)$"
|
||||
r"|(^|/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|uv\.lock|poetry\.lock|Cargo\.lock)$"
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Finding:
|
||||
path: str
|
||||
line: int
|
||||
kind: str
|
||||
match: str
|
||||
|
||||
|
||||
def load_allow(path: Path = ALLOW_FILE) -> list[dict]:
|
||||
data = yaml.safe_load(path.read_text()) or {}
|
||||
entries = data.get("allow") or []
|
||||
for e in entries: # a reason per entry is the whole point of the file
|
||||
if not (e.get("repo") and e.get("paths") and str(e.get("reason", "")).strip()):
|
||||
raise ValueError(f"allow entry needs repo, paths and a reason: {e}")
|
||||
return entries
|
||||
|
||||
|
||||
def allowed(repo: str, path: str, allow: list[dict]) -> bool:
|
||||
for e in allow:
|
||||
if e["repo"] == repo and any(fnmatch.fnmatch(path, g) for g in e["paths"]):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
COMMENT = re.compile(r"^\s*(?:#|//|/\*|\*|<!--)")
|
||||
|
||||
|
||||
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
||||
# A comment is not a call: "the ANTHROPIC_OAUTH_TOKEN setting was removed"
|
||||
# (windy-search) must not count, nor a commented-out `# OPENAI_API_KEY=`.
|
||||
if COMMENT.match(text):
|
||||
return []
|
||||
hits = []
|
||||
for kind, rx in RULES:
|
||||
if kind == "provider SDK dep" and not DEP_FILES.search(path):
|
||||
continue
|
||||
m = rx.search(text)
|
||||
if m:
|
||||
hits.append((kind, m.group(0).strip()[:60]))
|
||||
return hits
|
||||
|
||||
|
||||
def _git(bare: Path, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
["git", "--git-dir", str(bare), *args],
|
||||
capture_output=True, text=True, check=True, timeout=120,
|
||||
).stdout
|
||||
|
||||
|
||||
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict]) -> list[Finding]:
|
||||
"""Every line in the tree at `sha` (default branch: the baseline)."""
|
||||
# A cheap prefilter by git, then the real rules in Python.
|
||||
pre = "|".join([re.escape(h) for h in HOSTS] + KEYS + ["anthropic", "openai", "groq", "mistral",
|
||||
"generativeai", "genai", "cohere", "together", "cerebras", "litellm"])
|
||||
try:
|
||||
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.returncode == 1: # no matches
|
||||
return []
|
||||
raise
|
||||
found = []
|
||||
for raw in out.splitlines():
|
||||
# <sha>:<path>:<line>:<text>
|
||||
try:
|
||||
_, path, line, text = raw.split(":", 3)
|
||||
except ValueError:
|
||||
continue
|
||||
if SKIP.search(path) or allowed(repo, path, allow):
|
||||
continue
|
||||
for kind, match in scan_line(path, text):
|
||||
found.append(Finding(path, int(line), kind, match))
|
||||
return found
|
||||
|
||||
|
||||
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict]) -> list[Finding]:
|
||||
"""Only the lines a PR adds, vs its merge-base with the default branch."""
|
||||
mb = _git(bare, "merge-base", base_ref, sha).strip()
|
||||
diff = _git(bare, "diff", "-U0", "--no-color", "--no-ext-diff", mb, sha)
|
||||
return parse_added(repo, diff, allow)
|
||||
|
||||
|
||||
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
|
||||
|
||||
def parse_added(repo: str, diff: str, allow: list[dict]) -> list[Finding]:
|
||||
found, path, line = [], None, 0
|
||||
for raw in diff.splitlines():
|
||||
if raw.startswith("+++ "):
|
||||
p = raw[4:]
|
||||
path = None if p == "/dev/null" else p[2:] if p.startswith("b/") else p
|
||||
continue
|
||||
m = HUNK.match(raw)
|
||||
if m:
|
||||
line = int(m.group(1))
|
||||
continue
|
||||
if path is None or raw.startswith("--- "):
|
||||
continue
|
||||
if raw.startswith("+"):
|
||||
if not (SKIP.search(path) or allowed(repo, path, allow)):
|
||||
for kind, match in scan_line(path, raw[1:]):
|
||||
found.append(Finding(path, line, kind, match))
|
||||
line += 1
|
||||
return found
|
||||
|
||||
|
||||
# ---- cache: a tree scan runs once per (repo, sha, rules+allow) --------------
|
||||
def _fingerprint(allow: list[dict]) -> str:
|
||||
return hashlib.sha256(
|
||||
json.dumps([HOSTS, KEYS, PY_SDKS, JS_SDKS, SKIP.pattern, allow], sort_keys=True).encode()
|
||||
).hexdigest()[:16]
|
||||
|
||||
|
||||
def cached_scan(key: str, fn) -> list[Finding]:
|
||||
try:
|
||||
cache = json.loads(CACHE.read_text())
|
||||
except (OSError, ValueError):
|
||||
cache = {}
|
||||
if key in cache:
|
||||
return [Finding(**f) for f in cache[key]]
|
||||
result = fn()
|
||||
cache[key] = [f.__dict__ for f in result]
|
||||
if len(cache) > 2000: # keep it small: newest entries win
|
||||
cache = dict(list(cache.items())[-1000:])
|
||||
try:
|
||||
CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = CACHE.with_suffix(".tmp")
|
||||
tmp.write_text(json.dumps(cache))
|
||||
tmp.replace(CACHE)
|
||||
except OSError:
|
||||
pass
|
||||
return result
|
||||
|
||||
|
||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> list[Finding] | None:
|
||||
"""Findings for one commit, or None when the guard can't run (never a fake OK)."""
|
||||
bare = WORK / f"{repo}.git"
|
||||
if not bare.is_dir():
|
||||
return None
|
||||
allow = load_allow()
|
||||
fp = _fingerprint(allow)
|
||||
if is_default_head:
|
||||
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
||||
return cached_scan(
|
||||
f"pr:{repo}:{sha}:{fp}",
|
||||
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||
)
|
||||
|
||||
|
||||
def status_for(findings: list[Finding], whole_tree: bool) -> tuple[str, str, Finding | None]:
|
||||
"""(state, description, first finding) for the GitHub commit status."""
|
||||
scope = "in tree" if whole_tree else "added"
|
||||
if not findings:
|
||||
what = "no direct AI-provider use in tree" if whole_tree else "no direct AI-provider use added"
|
||||
return "success", f"OK: {what} (Windy Mind is the only door)", None
|
||||
f = findings[0]
|
||||
n = len(findings)
|
||||
state = "failure" if MODE == "block" else "success"
|
||||
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
||||
desc = f"{lead}: {n} direct AI-provider use{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"
|
||||
return state, desc[:140], f
|
||||
|
||||
|
||||
def report(repos: list[str]) -> int:
|
||||
allow = load_allow()
|
||||
total = 0
|
||||
for repo in repos:
|
||||
bare = WORK / f"{repo}.git"
|
||||
if not bare.is_dir():
|
||||
print(f"## {repo}: no sync clone, skipped")
|
||||
continue
|
||||
head = _git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||
sha = _git(bare, "rev-parse", head).strip()
|
||||
fs = scan_tree(repo, bare, sha, allow)
|
||||
total += len(fs)
|
||||
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} finding(s)")
|
||||
for f in fs:
|
||||
print(f" {f.path}:{f.line} [{f.kind}] {f.match}")
|
||||
print(f"TOTAL {total}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
||||
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
||||
p.name.removesuffix(".git") for p in WORK.glob("*.git"))
|
||||
sys.exit(report(sys.argv[2:] or default))
|
||||
sys.exit(__doc__)
|
||||
@@ -343,6 +343,37 @@ def post_statuses(repo: str, sha: str) -> None:
|
||||
print(f" {repo}@{sha[:7]} {ctx} = {state} -> {st}")
|
||||
|
||||
|
||||
GUARD_CTX = "windy-git/compute-guard"
|
||||
|
||||
|
||||
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only).
|
||||
|
||||
Non-fatal and never a fake OK: if the guard can't run, nothing is posted.
|
||||
"""
|
||||
try:
|
||||
import compute_guard as cg # same directory; loaded lazily so the bridge never depends on it
|
||||
|
||||
findings = cg.check(repo, sha, default_branch, is_default_head)
|
||||
except Exception as e: # noqa: BLE001 — the guard must never break CI signals
|
||||
print(f" {repo}@{sha[:7]} compute-guard skipped ({type(e).__name__}: {str(e)[:80]})")
|
||||
return
|
||||
if findings is None:
|
||||
return
|
||||
state, desc, first = cg.status_for(findings, whole_tree=is_default_head)
|
||||
st, existing = github("GET", f"/repos/{GH_OWNER}/{repo}/commits/{sha}/statuses?per_page=100")
|
||||
for s in existing or []: # newest first: compare the latest guard status only
|
||||
if s["context"] == GUARD_CTX:
|
||||
if (s["state"], s.get("description")) == (state, desc):
|
||||
return
|
||||
break
|
||||
url = (f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}/{first.path}#L{first.line}"
|
||||
if first else f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}")
|
||||
st, _ = github("POST", f"/repos/{GH_OWNER}/{repo}/statuses/{sha}",
|
||||
{"state": state, "context": GUARD_CTX, "description": desc, "target_url": url})
|
||||
print(f" {repo}@{sha[:7]} {GUARD_CTX} = {state} ({len(findings)} finding(s)) -> {st}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
if not (GITEA_TOKEN and GITHUB_TOKEN):
|
||||
sys.exit("GITEA_ADMIN_TOKEN and GITHUB_TOKEN are required")
|
||||
@@ -350,13 +381,17 @@ def main() -> int:
|
||||
for repo in REPOS:
|
||||
try:
|
||||
shas = sync_prs(repo)
|
||||
default_branch, default_head = "main", None
|
||||
st, br = github("GET", f"/repos/{GH_OWNER}/{repo}")
|
||||
if st == 200:
|
||||
st, b = github("GET", f"/repos/{GH_OWNER}/{repo}/branches/{br['default_branch']}")
|
||||
default_branch = br["default_branch"]
|
||||
st, b = github("GET", f"/repos/{GH_OWNER}/{repo}/branches/{default_branch}")
|
||||
if st == 200:
|
||||
shas.append(b["commit"]["sha"])
|
||||
default_head = b["commit"]["sha"]
|
||||
shas.append(default_head)
|
||||
for sha in dict.fromkeys(shas):
|
||||
post_statuses(repo, sha)
|
||||
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
||||
except Exception as e: # one repo's failure must not hide the others'
|
||||
print(f" FAILED {repo}: {e}")
|
||||
failed = 1
|
||||
|
||||
@@ -22,7 +22,11 @@ head=$($G rev-parse "refs/heads/${branch}")
|
||||
[[ "$head" == "$want"* ]] || { echo "refusing: ${branch} is at ${head:0:7}, not ${want}"; exit 1; }
|
||||
parent=$($G rev-parse "${head}^")
|
||||
|
||||
while systemctl is-active -q windygit-sync; do sleep 5; done
|
||||
# NOT `systemctl is-active`: the sync is Type=oneshot, which reads "activating"
|
||||
# (exit 3) for its whole run, so is-active says "idle" mid-run.
|
||||
busy() { case "$(systemctl show windygit-sync -p ActiveState --value)" in
|
||||
activating|active|deactivating|reloading) return 0;; esac; return 1; }
|
||||
while busy; do sleep 5; done
|
||||
$G update-ref "refs/heads/${branch}" "$parent" "$head"
|
||||
mark=$(awk '{print int($1*1000000)}' /proc/uptime)
|
||||
echo "rewound ${repo}:${branch} ${head:0:7} -> ${parent:0:7}"
|
||||
|
||||
Reference in New Issue
Block a user