Compare commits
15 Commits
telemetry-
...
87dcddb87d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87dcddb87d | ||
|
|
a5f7b036a8 | ||
|
|
255aa58b35 | ||
|
|
e64a1b5fcb | ||
|
|
1bc55eaec9 | ||
|
|
fdb0f5989e | ||
|
|
9be2952ff8 | ||
|
|
fe3bce39ff | ||
|
|
6b0b60eb4a | ||
|
|
60708dd8db | ||
|
|
aedc772d29 | ||
|
|
acb8ec3a16 | ||
|
|
0051c72037 | ||
|
|
4c76b1b12a | ||
|
|
2d4fadb090 |
@@ -51,9 +51,10 @@ jobs:
|
|||||||
- name: install
|
- name: install
|
||||||
run: |
|
run: |
|
||||||
python3 --version
|
python3 --version
|
||||||
python3 -m venv .venv
|
# From uv.lock, never floating: CI tests exactly what the image ships.
|
||||||
.venv/bin/pip install -q --upgrade pip
|
# --locked also FAILS if pyproject.toml changed without re-locking.
|
||||||
.venv/bin/pip install -e ".[dev]"
|
python3 -m pip install -q uv==0.12.5
|
||||||
|
uv sync --locked --extra dev
|
||||||
|
|
||||||
- name: lint
|
- name: lint
|
||||||
run: .venv/bin/ruff check api scripts
|
run: .venv/bin/ruff check api scripts
|
||||||
|
|||||||
15
Dockerfile
15
Dockerfile
@@ -10,10 +10,19 @@ WORKDIR /app
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends git curl \
|
RUN apt-get update && apt-get install -y --no-install-recommends git curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
COPY pyproject.toml ./
|
# Dependencies come from uv.lock, hash-pinned, never "latest at build time".
|
||||||
RUN pip install --no-cache-dir -e .
|
# Floating installs meant a rebuild could ship different fastapi/starlette/
|
||||||
|
# pydantic than CI tested (Windy Cloud's OpenAPI drift, 09-23). The lock was
|
||||||
|
# cut to exactly what prod ran then. uv only exports; pip installs, so the
|
||||||
|
# image layout (system python, uvicorn on PATH) is unchanged.
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv
|
||||||
|
COPY pyproject.toml uv.lock ./
|
||||||
|
RUN uv export --frozen --no-dev --no-emit-project -o /tmp/requirements.txt \
|
||||||
|
&& pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \
|
||||||
|
&& rm /tmp/requirements.txt
|
||||||
COPY api ./api
|
COPY api ./api
|
||||||
|
RUN pip install --no-cache-dir --no-deps -e .
|
||||||
|
|
||||||
COPY alembic ./alembic
|
COPY alembic ./alembic
|
||||||
COPY alembic.ini ./
|
COPY alembic.ini ./
|
||||||
COPY scripts ./scripts
|
COPY scripts ./scripts
|
||||||
|
|||||||
92
api/tests/test_canary_cleanup.py
Normal file
92
api/tests/test_canary_cleanup.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""The canary's login probe must end the session it opens (journey cleanup rule)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_spec = importlib.util.spec_from_file_location("canary", ROOT / "scripts" / "canary.py")
|
||||||
|
canary = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["canary"] = canary # dataclasses resolve their module by name
|
||||||
|
_spec.loader.exec_module(canary)
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, status=200, body=b"{}"):
|
||||||
|
self.status, self._body = status, body
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
return self._body
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _err(code):
|
||||||
|
return urllib.error.HTTPError(canary.LOGOUT_URL, code, "x", {}, io.BytesIO(b""))
|
||||||
|
|
||||||
|
|
||||||
|
def _script(monkeypatch, outcomes):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(req, timeout=None):
|
||||||
|
calls.append((req.get_method(), req.full_url, req.get_header("Authorization")))
|
||||||
|
o = outcomes.pop(0)
|
||||||
|
if isinstance(o, Exception):
|
||||||
|
raise o
|
||||||
|
return o
|
||||||
|
|
||||||
|
monkeypatch.setattr(canary.urllib.request, "urlopen", fake)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_logout_ends_the_session(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_Resp(200)])
|
||||||
|
r = canary.logout("tok", sleep=lambda s: None)
|
||||||
|
assert r.status == "ok"
|
||||||
|
assert calls == [("POST", canary.LOGOUT_URL, "Bearer tok")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_5xx_and_no_response_are_retried_then_succeed(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(502), OSError("reset"), _Resp(200)])
|
||||||
|
assert canary.logout("tok", sleep=lambda s: None).status == "ok"
|
||||||
|
assert len(calls) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_already_over_counts_as_done(monkeypatch):
|
||||||
|
_script(monkeypatch, [_err(401)])
|
||||||
|
assert canary.logout("tok", sleep=lambda s: None).status == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_other_4xx_fails_fast_and_honestly(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(400)])
|
||||||
|
r = canary.logout("tok", sleep=lambda s: None)
|
||||||
|
assert r.status == "down" and r.detail.startswith("CLEANUP FAILED") and len(calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_retries_are_bounded_and_reported(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(503)] * 8)
|
||||||
|
r = canary.logout("tok", attempts=8, sleep=lambda s: None)
|
||||||
|
assert r.status == "down" and "CLEANUP FAILED after 8 tries" in r.detail and len(calls) == 8
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_probe_logs_out_with_the_token_it_got(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_Resp(200, b'{"token": "abc"}'), _Resp(200)])
|
||||||
|
c = canary.Check("identity.login", "https://account.windyword.ai/api/v1/auth/login", "x",
|
||||||
|
method="POST", body={"email": "e", "password": "p"},
|
||||||
|
after=canary._logout_after_login)
|
||||||
|
r = canary._probe(c)
|
||||||
|
assert r.status == "ok" and [f.status for f in r.followups] == ["ok"]
|
||||||
|
assert calls[1] == ("POST", canary.LOGOUT_URL, "Bearer abc")
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_without_token_is_a_cleanup_failure_not_a_pass():
|
||||||
|
[f] = canary._logout_after_login(b"{}")
|
||||||
|
assert f.status == "down" and "CLEANUP FAILED" in f.detail
|
||||||
118
api/tests/test_ci_hygiene.py
Normal file
118
api/tests/test_ci_hygiene.py
Normal file
@@ -0,0 +1,118 @@
|
|||||||
|
"""CI hygiene guard (house rule 6): lockfile-only installs, no host-port services."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / "scripts"))
|
||||||
|
_spec = importlib.util.spec_from_file_location("ci_hygiene", ROOT / "scripts" / "ci_hygiene.py")
|
||||||
|
hy = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["ci_hygiene"] = hy
|
||||||
|
_spec.loader.exec_module(hy)
|
||||||
|
|
||||||
|
WF = ".github/workflows/ci.yml"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path, text", [
|
||||||
|
(WF, " .venv/bin/pip install -e \".[dev]\""), # windy-git's own, before e64a1b5
|
||||||
|
("Dockerfile", "RUN pip install --no-cache-dir -e ."), # windy-git image, before e64a1b5
|
||||||
|
(WF, " - run: uv pip install -e \".[dev]\""), # WindyCloud #109's CI
|
||||||
|
(WF, " run: pip install fastapi uvicorn"),
|
||||||
|
(WF, " - run: uv sync --all-extras"), # windy-mind style, not locked
|
||||||
|
(WF, " - run: npm install"), # windy-drops / windytalk
|
||||||
|
(WF, " - run: npm install --no-save --no-audit --no-fund jsdom"), # windy-pro reality-check
|
||||||
|
(WF, " - run: yarn install"),
|
||||||
|
(WF, " - run: cd web && pnpm install"),
|
||||||
|
("docker/api.Dockerfile", "RUN apt-get update && pip install requests"),
|
||||||
|
])
|
||||||
|
def test_floating_installs_are_flagged(path, text):
|
||||||
|
assert [k for k, _ in hy.scan_line(path, text)] == ["floating install"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path, text", [
|
||||||
|
(WF, " python3 -m pip install -q uv==0.12.5"), # exact tool pin
|
||||||
|
(WF, " uv sync --locked --extra dev"),
|
||||||
|
(WF, " - run: uv sync --frozen"),
|
||||||
|
(WF, " - run: npm ci"),
|
||||||
|
(WF, " - run: npm install --no-save jsdom@24.1.0"),
|
||||||
|
(WF, " - run: pip install -r requirements.lock --require-hashes"),
|
||||||
|
(WF, " - run: pip install -r requirements.txt"),
|
||||||
|
("Dockerfile", " && pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \\\\"),
|
||||||
|
("Dockerfile", "RUN pip install --no-cache-dir --no-deps -e ."), # project only, deps from the lock
|
||||||
|
(WF, " .venv/bin/pip install -q --upgrade pip"),
|
||||||
|
(WF, " - run: yarn install --frozen-lockfile"),
|
||||||
|
(WF, " # - run: npm install (commented out)"),
|
||||||
|
(WF, " - run: echo 'pip is great'"),
|
||||||
|
])
|
||||||
|
def test_locked_or_pinned_installs_pass(path, text):
|
||||||
|
assert hy.scan_line(path, text) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text, port", [
|
||||||
|
(" - 5432:5432", "5432"), # windy-mind / eternitas (collided 09-23)
|
||||||
|
(" - '15432:5432'", "15432"), # WindyCloud
|
||||||
|
(' - "6379:6379"', "6379"),
|
||||||
|
])
|
||||||
|
def test_services_publishing_a_host_port_are_flagged(text, port):
|
||||||
|
[(kind, match)] = hy.scan_line(WF, text)
|
||||||
|
assert kind == "host port" and port in match
|
||||||
|
|
||||||
|
|
||||||
|
def test_host_port_rule_is_for_workflows_only():
|
||||||
|
assert hy.scan_line("docker-compose.yml", " - 5432:5432") == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path, ok", [
|
||||||
|
(".github/workflows/ci.yml", True), (".gitea/workflows/check.yaml", True),
|
||||||
|
("Dockerfile", True), ("api/Dockerfile.prod", True), ("docker/web.Dockerfile", True),
|
||||||
|
("scripts/setup.sh", False), ("README.md", False), ("node_modules/x/Dockerfile", False),
|
||||||
|
(".github/lint/x.yml", False),
|
||||||
|
])
|
||||||
|
def test_scope_is_ci_workflows_and_dockerfiles(path, ok):
|
||||||
|
assert hy.path_ok(path) is ok
|
||||||
|
|
||||||
|
|
||||||
|
def test_warn_mode_never_turns_red(monkeypatch):
|
||||||
|
monkeypatch.setattr(hy, "MODE", "warn")
|
||||||
|
state, desc, f = hy.status_for([hy.cg.Finding(WF, 12, "floating install", "npm install (use npm ci)")], True)
|
||||||
|
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 CI hygiene issue in CI/Dockerfiles")
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_file_loads_and_is_empty_today():
|
||||||
|
assert hy.cg.load_allow(hy.ALLOW_FILE) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path, text, want", [
|
||||||
|
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv", "ghcr.io/astral-sh/uv:latest"), # Mail #147
|
||||||
|
("Dockerfile", "FROM python:latest", "python:latest"),
|
||||||
|
("Dockerfile", "FROM --platform=linux/amd64 node:latest AS web", "node:latest"),
|
||||||
|
(WF, " image: postgres:latest", "postgres:latest"),
|
||||||
|
(WF, " - uses: docker://ghcr.io/foo/bar:latest", "ghcr.io/foo/bar:latest"),
|
||||||
|
])
|
||||||
|
def test_latest_images_are_flagged(path, text, want):
|
||||||
|
hits = hy.scan_line(path, text)
|
||||||
|
assert ("floating image", want) in hits
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("text", [
|
||||||
|
"COPY pyproject.toml uv.lock* ./", # Windy Mail #147
|
||||||
|
"COPY package.json package-lock.json* ./",
|
||||||
|
])
|
||||||
|
def test_optional_lock_globs_are_flagged(text):
|
||||||
|
assert [k for k, _ in hy.scan_line("Dockerfile", text)] == ["optional lock"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("path, text", [
|
||||||
|
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv"),
|
||||||
|
("Dockerfile", "FROM python:3.12-slim"),
|
||||||
|
("Dockerfile", "COPY pyproject.toml uv.lock ./"),
|
||||||
|
("Dockerfile", "COPY src/*.py ./src/"),
|
||||||
|
("Dockerfile", "RUN echo latest release notes"),
|
||||||
|
])
|
||||||
|
def test_pinned_images_and_real_locks_pass(path, text):
|
||||||
|
assert hy.scan_line(path, text) == []
|
||||||
193
api/tests/test_compute_guard.py
Normal file
193
api/tests/test_compute_guard.py
Normal file
@@ -0,0 +1,193 @@
|
|||||||
|
"""Compute guard: Windy Mind is the only door to AI compute (warn-only today)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_spec = importlib.util.spec_from_file_location("compute_guard", ROOT / "scripts" / "compute_guard.py")
|
||||||
|
cg = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["compute_guard"] = cg
|
||||||
|
_spec.loader.exec_module(cg)
|
||||||
|
|
||||||
|
ALLOW = cg.load_allow(ROOT / "ci" / "compute-guard-allow.yml")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"path, text, kind",
|
||||||
|
[
|
||||||
|
# audit #1 (windy-search, closed) and #2 (windy-chat, live): the shapes they had
|
||||||
|
("service/app/anthropic_client.py", 'URL = "https://api.anthropic.com/v1/messages"', "provider host"),
|
||||||
|
("service/app/config.py", 'token = os.environ["ANTHROPIC_OAUTH_TOKEN"]', "provider key"),
|
||||||
|
("services/agent-roster/lib/llm.js", "const url = 'https://api.groq.com/openai/v1/chat/completions'", "provider host"),
|
||||||
|
("docker-compose.yml", " GROQ_API_KEY: ${GROQ_API_KEY}", "provider key"),
|
||||||
|
# audit #3/#4 (windy-pro account-server)
|
||||||
|
("account-server/src/routes/transcription.ts", "const r = await fetch('https://api.openai.com/v1/audio/transcriptions'", "provider host"),
|
||||||
|
("account-server/src/config.ts", "openaiKey: process.env.OPENAI_API_KEY,", "provider key"),
|
||||||
|
# SDKs and deps
|
||||||
|
("app/llm.py", "from anthropic import Anthropic", "provider SDK"),
|
||||||
|
("app/llm.py", "import openai", "provider SDK"),
|
||||||
|
("app/llm.py", "import google.generativeai as genai", "provider SDK"),
|
||||||
|
("src/ai.ts", 'import Anthropic from "@anthropic-ai/sdk";', "provider SDK"),
|
||||||
|
("src/ai.js", "const Groq = require('groq-sdk')", "provider SDK"),
|
||||||
|
("package.json", ' "openai": "^4.52.0",', "provider SDK dep"),
|
||||||
|
("requirements.txt", "anthropic>=0.40", "provider SDK dep"),
|
||||||
|
("pyproject.toml", ' "google-generativeai>=0.8",', "provider SDK dep"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_audit_shapes_are_flagged(path, text, kind):
|
||||||
|
assert kind in [k for k, _ in cg.scan_line(path, text)]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"path, text",
|
||||||
|
[
|
||||||
|
("app/mind.py", 'MIND = "https://mind.windyword.ai/v1/chat/completions"'), # the door itself
|
||||||
|
("app/models.py", "openai_compatible = True # Mind speaks the OpenAI wire format"),
|
||||||
|
("app/x.py", "from app.openai_shim import x"), # a local module, not the SDK
|
||||||
|
("package.json", ' "openai-types-lite": "1.0.0",'), # a different package
|
||||||
|
("README.txt", "set OPENAI_API_KEY"), # scanned-by-rule, excluded by SKIP separately
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_near_misses_are_not_flagged(path, text):
|
||||||
|
if cg.SKIP.search(path):
|
||||||
|
return
|
||||||
|
assert cg.scan_line(path, text) == []
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"path",
|
||||||
|
["tests/test_llm.py", "api/tests/x.py", "src/ai.test.ts", "web/foo.spec.js", "docs/setup.md",
|
||||||
|
"README.md", "package-lock.json", "uv.lock", "node_modules/openai/index.js", ".github/workflows/ci.yml",
|
||||||
|
"conftest.py", "app/llm_test.py"],
|
||||||
|
)
|
||||||
|
def test_tests_docs_lockfiles_vendored_ci_are_never_scanned(path):
|
||||||
|
assert cg.SKIP.search(path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_allow_list_needs_a_reason_per_entry(tmp_path):
|
||||||
|
bad = tmp_path / "a.yml"
|
||||||
|
bad.write_text("allow:\n - repo: x\n paths: ['*']\n")
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
cg.load_allow(bad)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"repo, path, ok",
|
||||||
|
[
|
||||||
|
("windy-mind", "app/providers/anthropic.py", True),
|
||||||
|
("windy-agent", "agent/providers.py", True),
|
||||||
|
("windy-code", "extensions/windy-ai/src/aiProvider.ts", True),
|
||||||
|
("windy-code", "web/server/llm.ts", False), # BYOK is the extension only
|
||||||
|
("windy-connect", "backend/src/writers/claude_code.py", True),
|
||||||
|
("windy-chat", "services/agent-roster/lib/llm.js", False), # audit #2: must be flagged
|
||||||
|
("windy-pro", "account-server/src/routes/translations.ts", False),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_allow_list_entries(repo, path, ok):
|
||||||
|
assert cg.allowed(repo, path, ALLOW) is ok
|
||||||
|
|
||||||
|
|
||||||
|
DIFF = """diff --git a/app/llm.py b/app/llm.py
|
||||||
|
--- a/app/llm.py
|
||||||
|
+++ b/app/llm.py
|
||||||
|
@@ -10,0 +11,2 @@
|
||||||
|
+import anthropic
|
||||||
|
+client = anthropic.Anthropic()
|
||||||
|
diff --git a/tests/test_llm.py b/tests/test_llm.py
|
||||||
|
--- /dev/null
|
||||||
|
+++ b/tests/test_llm.py
|
||||||
|
@@ -0,0 +1 @@
|
||||||
|
+import anthropic
|
||||||
|
@@ -40 +42 @@
|
||||||
|
-x = 1
|
||||||
|
+x = 2
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_added_non_test_lines_are_findings():
|
||||||
|
fs = cg.parse_added("windy-chat", DIFF, ALLOW)
|
||||||
|
assert [(f.path, f.line, f.kind) for f in fs] == [("app/llm.py", 11, "provider SDK")]
|
||||||
|
|
||||||
|
|
||||||
|
def _repo(tmp_path, files: dict[str, str]) -> tuple[Path, str]:
|
||||||
|
work = tmp_path / "w"
|
||||||
|
work.mkdir()
|
||||||
|
run = lambda *a: subprocess.run(["git", *a], cwd=work, check=True, capture_output=True) # noqa: E731
|
||||||
|
run("init", "-q", "-b", "main")
|
||||||
|
for p, text in files.items():
|
||||||
|
(work / p).parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
(work / p).write_text(text)
|
||||||
|
run("add", "-A")
|
||||||
|
run("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "x")
|
||||||
|
bare = tmp_path / "r.git"
|
||||||
|
subprocess.run(["git", "clone", "-q", "--bare", str(work), str(bare)], check=True)
|
||||||
|
sha = subprocess.run(["git", "--git-dir", str(bare), "rev-parse", "main"],
|
||||||
|
capture_output=True, text=True, check=True).stdout.strip()
|
||||||
|
return bare, sha
|
||||||
|
|
||||||
|
|
||||||
|
def test_tree_scan_on_a_real_git_repo(tmp_path):
|
||||||
|
bare, sha = _repo(tmp_path, {
|
||||||
|
"app/llm.py": "import os\nKEY = os.environ['OPENAI_API_KEY']\n",
|
||||||
|
"app/ok.py": "MIND = 'https://mind.windyword.ai'\n",
|
||||||
|
"tests/test_llm.py": "import anthropic\n",
|
||||||
|
"docs/x.md": "api.anthropic.com\n",
|
||||||
|
})
|
||||||
|
fs = cg.scan_tree("windy-chat", bare, sha, ALLOW)
|
||||||
|
assert [(f.path, f.line, f.kind) for f in fs] == [("app/llm.py", 2, "provider key")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_warn_mode_never_turns_red(monkeypatch):
|
||||||
|
monkeypatch.setattr(cg, "MODE", "warn")
|
||||||
|
state, desc, f = cg.status_for([cg.Finding("a.py", 3, "provider host", "api.openai.com")], whole_tree=False)
|
||||||
|
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 direct AI-provider use added")
|
||||||
|
assert "a.py:3" in desc and f.path == "a.py"
|
||||||
|
|
||||||
|
|
||||||
|
def test_block_mode_fails(monkeypatch):
|
||||||
|
monkeypatch.setattr(cg, "MODE", "block")
|
||||||
|
state, desc, _ = cg.status_for([cg.Finding("a.py", 3, "provider host", "x")], whole_tree=True)
|
||||||
|
assert state == "failure" and desc.startswith("BLOCKED")
|
||||||
|
|
||||||
|
|
||||||
|
def test_clean_is_ok():
|
||||||
|
assert cg.status_for([], whole_tree=True)[:2] == (
|
||||||
|
"success", "OK: no direct AI-provider use in tree (Windy Mind is the only door)")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"text",
|
||||||
|
[
|
||||||
|
" # The ANTHROPIC_OAUTH_TOKEN setting was removed on 2026-09-23 ON PURPOSE", # windy-search
|
||||||
|
"# ANTHROPIC_API_KEY=",
|
||||||
|
" // fallback used to call https://api.groq.com directly",
|
||||||
|
" * @see https://api.openai.com/v1/audio",
|
||||||
|
"<!-- api.anthropic.com -->",
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_comments_are_not_calls(text):
|
||||||
|
assert cg.scan_line("service/app/config.py", text) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_code_with_a_trailing_comment_still_counts():
|
||||||
|
assert cg.scan_line("a.js", "fetch('https://api.openai.com/v1') // TODO move to Mind")
|
||||||
|
|
||||||
|
|
||||||
|
def test_windy_pro_desktop_is_byok_but_the_account_server_is_not():
|
||||||
|
assert cg.allowed("windy-pro", "src/client/desktop/main.js", ALLOW)
|
||||||
|
assert not cg.allowed("windy-pro", "account-server/src/routes/translations.ts", ALLOW)
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_commit_not_fetched_yet_is_skipped_not_an_error(tmp_path, monkeypatch):
|
||||||
|
bare, sha = _repo(tmp_path, {"app/llm.py": "import anthropic\n"})
|
||||||
|
monkeypatch.setattr(cg, "WORK", tmp_path)
|
||||||
|
monkeypatch.setattr(cg, "CACHE", tmp_path / "cache.json")
|
||||||
|
(tmp_path / "windy-chat.git").symlink_to(bare)
|
||||||
|
assert cg.check("windy-chat", "f" * 40, "main", True) is None # pushed after the fetch
|
||||||
|
assert [f.kind for f in cg.check("windy-chat", sha, "main", True)] == ["provider SDK"]
|
||||||
61
api/tests/test_guards_report.py
Normal file
61
api/tests/test_guards_report.py
Normal file
@@ -0,0 +1,61 @@
|
|||||||
|
"""guards_report: job attribution and the Grant-owned split."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / "scripts"))
|
||||||
|
_spec = importlib.util.spec_from_file_location("guards_report", ROOT / "scripts" / "guards_report.py")
|
||||||
|
gr = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["guards_report"] = gr
|
||||||
|
_spec.loader.exec_module(gr)
|
||||||
|
|
||||||
|
OWNED = yaml.safe_load((ROOT / "ci" / "grant-owned.yml").read_text())["grant_owned"]
|
||||||
|
WF = """name: CI
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
jobs:
|
||||||
|
reality-check:
|
||||||
|
runs-on: x
|
||||||
|
steps:
|
||||||
|
- run: npm install jsdom
|
||||||
|
test-backend:
|
||||||
|
runs-on: x
|
||||||
|
steps:
|
||||||
|
- run: pip install pytest
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_job_of_attributes_lines_to_their_job():
|
||||||
|
assert gr.job_of(WF, 3) is None # `on:` block, not a job
|
||||||
|
assert gr.job_of(WF, 8) == "reality-check"
|
||||||
|
assert gr.job_of(WF, 12) == "test-backend"
|
||||||
|
|
||||||
|
|
||||||
|
def test_windy_pro_desktop_jobs_and_paths_are_grant_owned():
|
||||||
|
ci = ".github/workflows/ci.yml"
|
||||||
|
assert gr.grant_owned("windy-pro", ci, "reality-check", OWNED)
|
||||||
|
assert gr.grant_owned("windy-pro", ci, "build-electron", OWNED)
|
||||||
|
assert not gr.grant_owned("windy-pro", ci, "test-backend", OWNED) # server side: 8c
|
||||||
|
assert gr.grant_owned("windy-pro", ".github/workflows/release-mac.yml", None, OWNED)
|
||||||
|
assert gr.grant_owned("windy-pro", "src/client/desktop/main.js", None, OWNED)
|
||||||
|
assert not gr.grant_owned("windy-pro", "services/account-server/Dockerfile", None, OWNED)
|
||||||
|
assert not gr.grant_owned("windy-chat", "src/client/desktop/main.js", None, OWNED)
|
||||||
|
|
||||||
|
|
||||||
|
def test_render_splits_lane_and_grant_counts():
|
||||||
|
F = gr.cg.Finding
|
||||||
|
res = {"windy-pro": {"sha": "a" * 40, "compute": [],
|
||||||
|
"hygiene": [(F("ci.yml", 8, "floating install", "npm install"), "reality-check", True),
|
||||||
|
(F("ci.yml", 12, "floating install", "pip x"), "test-backend", False)]},
|
||||||
|
"windy-git": {"sha": "b" * 40, "compute": [], "hygiene": []}}
|
||||||
|
md = gr.render(res)
|
||||||
|
assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md
|
||||||
|
assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md
|
||||||
|
assert "| windy-git | bbbbbbb | 0 | 0 | clean ✅ |" in md
|
||||||
|
assert "(job reality-check)" in md
|
||||||
@@ -10,6 +10,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import base64
|
import base64
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -79,10 +80,11 @@ class Fake:
|
|||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def fake(monkeypatch):
|
def fake(monkeypatch):
|
||||||
def make(**kw):
|
def make(queued=(), **kw):
|
||||||
f = Fake(**kw)
|
f = Fake(**kw)
|
||||||
monkeypatch.setattr(bridge, "gitea", f.gitea)
|
monkeypatch.setattr(bridge, "gitea", f.gitea)
|
||||||
monkeypatch.setattr(bridge, "github", f.github)
|
monkeypatch.setattr(bridge, "github", f.github)
|
||||||
|
monkeypatch.setattr(bridge, "queued_jobs", lambda repo, sha: list(queued))
|
||||||
return f
|
return f
|
||||||
|
|
||||||
return make
|
return make
|
||||||
@@ -272,3 +274,119 @@ def test_gitea_dir_wins_over_github_dir(fake):
|
|||||||
)
|
)
|
||||||
def test_workflow_problem(text, problem):
|
def test_workflow_problem(text, problem):
|
||||||
assert bridge.workflow_problem(text) == problem
|
assert bridge.workflow_problem(text) == problem
|
||||||
|
|
||||||
|
|
||||||
|
def _q(n, wf, job):
|
||||||
|
return {"run_number": n, "workflow_id": wf, "name": job}
|
||||||
|
|
||||||
|
|
||||||
|
def test_queued_job_shows_pending_instead_of_nothing(fake):
|
||||||
|
f = fake(queued=[_q(5, "ci.yml", "test")])
|
||||||
|
bridge.post_statuses("windy-chat", SHA)
|
||||||
|
assert [(p["context"], p["state"]) for p in f.posted] == [("windy-git/ci/test", "pending")]
|
||||||
|
assert f.posted[0]["target_url"].endswith("/actions/runs/5")
|
||||||
|
|
||||||
|
|
||||||
|
def test_queued_rerun_supersedes_the_stale_failure(fake):
|
||||||
|
f = fake(runs=[_run(1, "ci.yml", "test", "failure", n=4)], queued=[_q(7, "ci.yml", "test")])
|
||||||
|
bridge.post_statuses("windy-chat", SHA)
|
||||||
|
assert [(p["context"], p["state"]) for p in f.posted] == [("windy-git/ci/test", "pending")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_older_queued_job_never_overrides_a_newer_verdict(fake):
|
||||||
|
f = fake(runs=[_run(1, "ci.yml", "test", "success", n=9)], queued=[_q(3, "ci.yml", "test")])
|
||||||
|
bridge.post_statuses("windy-chat", SHA)
|
||||||
|
assert [(p["context"], p["state"]) for p in f.posted] == [("windy-git/ci/test", "success")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_queued_docker_and_non_blocking_jobs_stay_unposted(fake):
|
||||||
|
f = fake(queued=[_q(2, "ci.yml", "docker-build"), _q(2, "ci.yml", "build-desktop")])
|
||||||
|
bridge.post_statuses("windy-pro", SHA)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_queued_lookup_refuses_unsafe_input():
|
||||||
|
assert bridge.queued_jobs("x'; drop table t;--", SHA) == []
|
||||||
|
assert bridge.queued_jobs("windy-chat", "not-a-sha") == []
|
||||||
|
|
||||||
|
|
||||||
|
def _db(monkeypatch, jobs, labels):
|
||||||
|
import json as _json
|
||||||
|
import subprocess as _sp
|
||||||
|
|
||||||
|
payload = _json.dumps({"jobs": jobs, "labels": [_json.dumps(x) for x in labels]})
|
||||||
|
monkeypatch.setattr(
|
||||||
|
bridge.subprocess, "run",
|
||||||
|
lambda *a, **k: _sp.CompletedProcess(a, 0, stdout=payload, stderr=""),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
RUNNER = ["veron-1", "linux-x64", "self-hosted", "linux", "x64"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_only_jobs_a_runner_can_take_are_pending(monkeypatch):
|
||||||
|
# macos-latest is cancelled unpicked by the janitor: pending would never resolve.
|
||||||
|
_db(monkeypatch, [
|
||||||
|
{"run_number": 3, "workflow_id": "ci.yml", "name": "test", "runs_on": '["self-hosted","linux","x64"]'},
|
||||||
|
{"run_number": 3, "workflow_id": "ci.yml", "name": "mac", "runs_on": '["macos-latest"]'},
|
||||||
|
], [RUNNER])
|
||||||
|
assert [j["name"] for j in bridge.queued_jobs("windy-chat", SHA)] == ["test"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_lookup_failure_is_non_fatal(monkeypatch):
|
||||||
|
import subprocess as _sp
|
||||||
|
|
||||||
|
def boom(*a, **k):
|
||||||
|
raise _sp.TimeoutExpired("docker", 30)
|
||||||
|
|
||||||
|
monkeypatch.setattr(bridge.subprocess, "run", boom)
|
||||||
|
assert bridge.queued_jobs("windy-chat", SHA) == []
|
||||||
|
|
||||||
|
|
||||||
|
class _Guard:
|
||||||
|
def __init__(self, findings):
|
||||||
|
self.findings = findings
|
||||||
|
|
||||||
|
def check(self, repo, sha, default_branch, is_default_head):
|
||||||
|
return self.findings
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def status_for(findings, whole_tree):
|
||||||
|
if not findings:
|
||||||
|
return "success", "OK: clean", None
|
||||||
|
return "success", f"WARN {len(findings)}", findings[0]
|
||||||
|
|
||||||
|
|
||||||
|
class _F:
|
||||||
|
path, line = "app/llm.py", 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_posts_warn_with_a_link_to_the_first_finding(fake, monkeypatch):
|
||||||
|
f = fake()
|
||||||
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||||
|
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||||
|
assert [(p["context"], p["state"], p["description"]) for p in f.posted] == [
|
||||||
|
("windy-git/compute-guard", "success", "WARN 1")]
|
||||||
|
assert f.posted[0]["target_url"].endswith(f"/src/commit/{SHA}/app/llm.py#L7")
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_same_status_is_not_reposted(fake, monkeypatch):
|
||||||
|
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "success", "description": "WARN 1"}])
|
||||||
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||||
|
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_guard_that_cannot_run_posts_nothing(fake, monkeypatch):
|
||||||
|
f = fake()
|
||||||
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard(None))
|
||||||
|
bridge.post_compute_guard("windy-chat", SHA, "main", True)
|
||||||
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_ci_hygiene_posts_under_its_own_context(fake, monkeypatch):
|
||||||
|
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "success", "description": "WARN 1"}])
|
||||||
|
monkeypatch.setitem(sys.modules, "ci_hygiene", _Guard([_F()]))
|
||||||
|
bridge.post_ci_hygiene("windy-chat", SHA, "main", True)
|
||||||
|
# the compute-guard status with the same description must not suppress it
|
||||||
|
assert [(p["context"], p["description"]) for p in f.posted] == [("windy-git/ci-hygiene", "WARN 1")]
|
||||||
|
|||||||
82
api/tests/test_push_velocity.py
Normal file
82
api/tests/test_push_velocity.py
Normal file
@@ -0,0 +1,82 @@
|
|||||||
|
"""Push-velocity detection (scripts/telemetry_emit.py): detect + alert only.
|
||||||
|
|
||||||
|
Driven through the real function with rows shaped like the Gitea query's.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
sys.path.insert(0, str(ROOT / "scripts"))
|
||||||
|
_spec = importlib.util.spec_from_file_location("telemetry_emit", ROOT / "scripts" / "telemetry_emit.py")
|
||||||
|
te = importlib.util.module_from_spec(_spec)
|
||||||
|
_spec.loader.exec_module(te)
|
||||||
|
|
||||||
|
NOW = 1_800_000_000.0
|
||||||
|
|
||||||
|
|
||||||
|
def row(login="agent-et26abcd1234", uid=7, p1h=0, p24h=0, d24h=0, repos=1, wid=None):
|
||||||
|
return {"uid": uid, "login": login, "wid": wid, "p1h": p1h, "p24h": p24h, "d24h": d24h, "repos": repos}
|
||||||
|
|
||||||
|
|
||||||
|
def test_under_every_threshold_emits_nothing():
|
||||||
|
ev, keep = te.push_velocity_events([row(p1h=60, p24h=500, d24h=10)], NOW, {})
|
||||||
|
assert ev == [] and keep == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_burst_emits_one_declared_row_with_the_passport():
|
||||||
|
ev, keep = te.push_velocity_events([row(p1h=61, p24h=61, repos=3)], NOW, {})
|
||||||
|
assert len(ev) == 1
|
||||||
|
e = ev[0]
|
||||||
|
assert e["event_type"] == "forge.push_velocity" and e["service"] == "forge"
|
||||||
|
assert e["actor_type"] == "agent" and e["actor_id"] == "ET26-ABCD-1234"
|
||||||
|
assert e["metadata"] == {
|
||||||
|
"rule": "pushes_1h", "window_s": 3600, "count": 61, "threshold": 60,
|
||||||
|
"repos": 3, "gitea_user_id": 7,
|
||||||
|
}
|
||||||
|
assert keep == {"7:pushes_1h": NOW}
|
||||||
|
|
||||||
|
|
||||||
|
def test_still_over_is_reported_once_per_window_not_every_run():
|
||||||
|
_, keep = te.push_velocity_events([row(p1h=90)], NOW, {})
|
||||||
|
ev, keep = te.push_velocity_events([row(p1h=95)], NOW + 300, keep)
|
||||||
|
assert ev == [] and keep == {"7:pushes_1h": NOW}
|
||||||
|
ev, _ = te.push_velocity_events([row(p1h=95)], NOW + 3601, keep)
|
||||||
|
assert len(ev) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_dropping_back_under_rearms():
|
||||||
|
_, keep = te.push_velocity_events([row(p1h=90)], NOW, {})
|
||||||
|
_, keep = te.push_velocity_events([row(p1h=5)], NOW + 300, keep)
|
||||||
|
assert keep == {}
|
||||||
|
ev, _ = te.push_velocity_events([row(p1h=90)], NOW + 600, keep)
|
||||||
|
assert len(ev) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_sync_account_is_exempt():
|
||||||
|
ev, _ = te.push_velocity_events([row(login="windyadmin", uid=1, p1h=9999, p24h=9999)], NOW, {})
|
||||||
|
assert ev == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_sso_human_is_keyed_on_windy_identity_id():
|
||||||
|
ev, _ = te.push_velocity_events([row(login="u-5e1b9569abc", wid="5e1b9569-full-id", d24h=11)], NOW, {})
|
||||||
|
assert [(e["actor_type"], e["actor_id"], e["metadata"]["rule"]) for e in ev] == [
|
||||||
|
("human", "5e1b9569-full-id", "ref_deletes_24h")
|
||||||
|
]
|
||||||
|
assert "caller" not in ev[0]["metadata"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_provable_id_is_system_plus_caller_never_an_invented_id():
|
||||||
|
# UPDATE 2 actor rule: agent/human rows without an actor_id are quarantined.
|
||||||
|
for login in ("u-nolink", "agent-weird"):
|
||||||
|
ev, _ = te.push_velocity_events([row(login=login, p24h=501)], NOW, {})
|
||||||
|
assert ev[0]["actor_type"] == "system" and "actor_id" not in ev[0]
|
||||||
|
assert ev[0]["metadata"]["caller"] == "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def test_passport_round_trip():
|
||||||
|
assert te.passport_from_login("agent-et26p1zgttp8") == "ET26-P1ZG-TTP8"
|
||||||
|
assert te.passport_from_login("u-abc") is None
|
||||||
5
ci/ci-hygiene-allow.yml
Normal file
5
ci/ci-hygiene-allow.yml
Normal file
@@ -0,0 +1,5 @@
|
|||||||
|
# CI hygiene allow-list: installs that may float, or services that may publish
|
||||||
|
# a host port. House rule 6 (09-23): installs come from a lockfile. Every entry
|
||||||
|
# is an exception and MUST say why. Paths are fnmatch globs from the repo root.
|
||||||
|
# Owner: Windy Git lane (13); changes go through the orchestrator.
|
||||||
|
allow: []
|
||||||
40
ci/compute-guard-allow.yml
Normal file
40
ci/compute-guard-allow.yml
Normal file
@@ -0,0 +1,40 @@
|
|||||||
|
# Compute guard allow-list: code that MAY talk to an AI provider directly.
|
||||||
|
# Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23). Every entry
|
||||||
|
# here is an exception to that rule and MUST say why. Paths are fnmatch globs
|
||||||
|
# relative to the repo root. Owner of this file: Windy Git lane (13); changes
|
||||||
|
# go through the orchestrator. Source of the first entries: COMPUTE_BYPASS_AUDIT.md.
|
||||||
|
allow:
|
||||||
|
- repo: windy-mind
|
||||||
|
paths: ["*"]
|
||||||
|
reason: "Windy Mind IS the door: provider clients belong here by definition."
|
||||||
|
|
||||||
|
- repo: windy-agent
|
||||||
|
paths: ["*"]
|
||||||
|
reason: >-
|
||||||
|
User BYOK: self-hosted agents call providers on the USER's own keys.
|
||||||
|
Mind stays opt-in there, or every self-hosted user's inference lands on
|
||||||
|
Grant's bill (no-cloud-cost-liability rule; audit #7).
|
||||||
|
|
||||||
|
- repo: windy-code
|
||||||
|
paths: ["extensions/windy-ai/*"]
|
||||||
|
reason: "User BYOK AI extension: the user's own provider keys; Mind is one opt-in provider (audit #8)."
|
||||||
|
|
||||||
|
- repo: windy-connect
|
||||||
|
paths: ["*writers/*"]
|
||||||
|
reason: "Writes client configs that NAME the user's own provider env vars; makes no provider calls (audit #11)."
|
||||||
|
|
||||||
|
- repo: windy-pro
|
||||||
|
paths: ["src/client/desktop/*"]
|
||||||
|
reason: >-
|
||||||
|
User BYOK desktop client: cloud STT/translate keys come from what the USER
|
||||||
|
enters (renderer localStorage -> electron-store; env var only for dev), and
|
||||||
|
the CSP line allows exactly those user-keyed hosts (audit #10). The
|
||||||
|
account-server is NOT covered: server-side calls go through Mind.
|
||||||
|
|
||||||
|
- repo: windy-pro
|
||||||
|
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
||||||
|
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
||||||
|
|
||||||
|
- repo: windy-git
|
||||||
|
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||||
|
reason: "The guard's own pattern list and this file."
|
||||||
15
ci/grant-owned.yml
Normal file
15
ci/grant-owned.yml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# Code Grant owns directly (orchestrator, 09-23): guard findings here are listed
|
||||||
|
# SEPARATELY in the guards status page and never hold up "block". Changes to
|
||||||
|
# these files are proposals for Grant / Windy Word 44, not a lane's fix.
|
||||||
|
grant_owned:
|
||||||
|
- repo: windy-pro
|
||||||
|
reason: "Windy Word desktop (Electron) + its release/installer builds: Grant's, built from the Mac mini."
|
||||||
|
paths:
|
||||||
|
- "src/client/desktop/*"
|
||||||
|
- "installer-v2/*"
|
||||||
|
- ".github/workflows/build-windows.yml"
|
||||||
|
- ".github/workflows/release-mac.yml"
|
||||||
|
- ".github/workflows/build-installer.yml"
|
||||||
|
- ".github/workflows/build-offline-installers.yml"
|
||||||
|
jobs:
|
||||||
|
".github/workflows/ci.yml": [reality-check, build-desktop, test-installer, build-electron]
|
||||||
15
deploy/runner/docker-compose.privileged.yml
Normal file
15
deploy/runner/docker-compose.privileged.yml
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
# ROLLBACK ONLY: the pre-Sysbox dind (privileged: true), kept one command away.
|
||||||
|
# Use it if CI breaks under Sysbox:
|
||||||
|
#
|
||||||
|
# cd /srv/windygit/src/deploy/runner
|
||||||
|
# sudo docker compose -f docker-compose.yml -f docker-compose.privileged.yml up -d dind
|
||||||
|
#
|
||||||
|
# (then restart the runners while idle). Compose merges `volumes` by container
|
||||||
|
# path, so this puts back the old `dind-storage` volume with its image cache.
|
||||||
|
# Going forward again: the same command without the second -f.
|
||||||
|
services:
|
||||||
|
dind:
|
||||||
|
runtime: runc
|
||||||
|
privileged: true
|
||||||
|
volumes:
|
||||||
|
- dind-storage:/var/lib/docker
|
||||||
@@ -26,8 +26,12 @@
|
|||||||
# * `dind` and every job container it spawns are UNTRUSTED. They are on a
|
# * `dind` and every job container it spawns are UNTRUSTED. They are on a
|
||||||
# private network with no access to the forge, its database, or its .env.
|
# private network with no access to the forge, its database, or its .env.
|
||||||
#
|
#
|
||||||
# dind itself is privileged — that is the cost, and it is the reason a job
|
# dind is NOT privileged (2026-09-23): it runs under the Sysbox runtime
|
||||||
# escape lands in a disposable daemon rather than on Grant's workstation.
|
# (sysbox-ce on Veron, `runtime: sysbox-runc`), a user-namespaced system
|
||||||
|
# container whose root is an unprivileged host uid. A job that escapes its own
|
||||||
|
# container lands in dind as a nobody on the host, not as root on Grant's
|
||||||
|
# workstation. Before Sysbox, dind was `privileged: true`; that config is kept
|
||||||
|
# as docker-compose.privileged.yml (ROLLBACK ONLY, one command, see that file).
|
||||||
#
|
#
|
||||||
# ⚠️ Do NOT "simplify" this by mounting the host docker socket.
|
# ⚠️ Do NOT "simplify" this by mounting the host docker socket.
|
||||||
|
|
||||||
@@ -36,13 +40,15 @@ name: windy-git-runner
|
|||||||
services:
|
services:
|
||||||
dind:
|
dind:
|
||||||
image: docker.io/library/docker:27-dind
|
image: docker.io/library/docker:27-dind
|
||||||
privileged: true
|
runtime: sysbox-runc # NOT privileged: see the I-5 note above
|
||||||
environment:
|
environment:
|
||||||
DOCKER_TLS_CERTDIR: "" # plain TCP on an isolated network, no host route
|
DOCKER_TLS_CERTDIR: "" # plain TCP on an isolated network, no host route
|
||||||
command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--tls=false"]
|
command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--tls=false"]
|
||||||
networks: [jobs]
|
networks: [jobs]
|
||||||
volumes:
|
volumes:
|
||||||
- dind-storage:/var/lib/docker
|
# A fresh volume: Sysbox shifts ownership to its own uid range. The old
|
||||||
|
# `dind-storage` is kept untouched for the privileged rollback.
|
||||||
|
- dind-storage-sysbox:/var/lib/docker
|
||||||
# G1.5 — bounded so a fork-bomb workflow cannot starve Grant's interactive
|
# G1.5 — bounded so a fork-bomb workflow cannot starve Grant's interactive
|
||||||
# session. Veron 1 is his workstation, not a dedicated build box.
|
# session. Veron 1 is his workstation, not a dedicated build box.
|
||||||
cpus: 12.0 # 12 of 24 cores
|
cpus: 12.0 # 12 of 24 cores
|
||||||
@@ -159,6 +165,7 @@ networks:
|
|||||||
|
|
||||||
volumes:
|
volumes:
|
||||||
dind-storage:
|
dind-storage:
|
||||||
|
dind-storage-sysbox:
|
||||||
runner-data:
|
runner-data:
|
||||||
runner-data-2:
|
runner-data-2:
|
||||||
runner-data-3:
|
runner-data-3:
|
||||||
|
|||||||
@@ -152,3 +152,16 @@ disqualifying the moment a stranger depends on it. **The trigger is not a date
|
|||||||
it is the first external push.** Move the control plane to a dedicated VPS (not
|
it is the first external push.** Move the control plane to a dedicated VPS (not
|
||||||
Kit 0), keep Veron 1 as the runner. It is an rsync, a Postgres dump and three
|
Kit 0), keep Veron 1 as the runner. It is an rsync, a Postgres dump and three
|
||||||
DNS record edits.
|
DNS record edits.
|
||||||
|
|
||||||
|
## Re-run a PR's CI (Gitea 1.24 has no rerun API)
|
||||||
|
|
||||||
|
```bash
|
||||||
|
ssh wg-veron
|
||||||
|
cd /srv/windygit/src && bash scripts/rerun_ci.sh <repo> <branch> <github-head-sha-prefix>
|
||||||
|
```
|
||||||
|
Moves the Windy Git branch back one commit; the next sync force-pushes the
|
||||||
|
GitHub head again and Gitea re-fires every workflow for that event on the same
|
||||||
|
commit. Guarded: refuses unless the branch is at the given sha, waits for a
|
||||||
|
sync that starts AFTER the rewind, restores the branch itself on timeout.
|
||||||
|
Don't use the web "Re-run" button: it needs a hub-SSO session as windyadmin,
|
||||||
|
which is Grant's identity.
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ import sys
|
|||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json")
|
STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json")
|
||||||
@@ -46,6 +47,16 @@ ALERT_FROM = os.environ.get("CANARY_ALERT_FROM", "office@thewindstorm.uk")
|
|||||||
LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35"))
|
LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35"))
|
||||||
TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60"))
|
TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60"))
|
||||||
|
|
||||||
|
# Journey cleanup rule (orchestrator, 2026-09-23). The login probe creates a hub
|
||||||
|
# session (access + refresh token) every run, so it must end it. The hub's
|
||||||
|
# /auth/logout revokes the token AND every refresh token of the account
|
||||||
|
# (verified live: access 401, refresh 401 after it). So the next successful
|
||||||
|
# logout also heals anything a failed run left behind; no ledger needed.
|
||||||
|
LOGOUT_URL = "https://account.windyword.ai/api/v1/auth/logout"
|
||||||
|
LOGOUT_ATTEMPTS = 8 # retried on 5xx / no response only
|
||||||
|
LOGOUT_GAP_S = 15.0
|
||||||
|
LOGOUT_GONE = (401, 404, 410) # the session is already over = done
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Result:
|
class Result:
|
||||||
@@ -54,6 +65,7 @@ class Result:
|
|||||||
detail: str
|
detail: str
|
||||||
seconds: float = 0.0
|
seconds: float = 0.0
|
||||||
user_visible: str = ""
|
user_visible: str = ""
|
||||||
|
followups: list[Result] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -68,6 +80,8 @@ class Check:
|
|||||||
# When True this check INVERTS: a 2xx is a critical failure (a security
|
# When True this check INVERTS: a 2xx is a critical failure (a security
|
||||||
# control opened) and a 401/403/503 is the healthy, expected outcome.
|
# control opened) and a 401/403/503 is the healthy, expected outcome.
|
||||||
must_refuse: bool = False
|
must_refuse: bool = False
|
||||||
|
# Runs on a 2xx with the response body; returns follow-up results (cleanup).
|
||||||
|
after: Callable[[bytes], list[Result]] | None = None
|
||||||
|
|
||||||
|
|
||||||
def _probe(c: Check) -> Result:
|
def _probe(c: Check) -> Result:
|
||||||
@@ -91,13 +105,18 @@ def _probe(c: Check) -> Result:
|
|||||||
elapsed, c.what_it_proves)
|
elapsed, c.what_it_proves)
|
||||||
if r.status >= 400:
|
if r.status >= 400:
|
||||||
return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves)
|
return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves)
|
||||||
|
raw = r.read()
|
||||||
warn = c.warn_seconds
|
warn = c.warn_seconds
|
||||||
if warn and elapsed > warn:
|
if warn and elapsed > warn:
|
||||||
return Result(
|
res = Result(
|
||||||
c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)",
|
c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)",
|
||||||
elapsed, c.what_it_proves,
|
elapsed, c.what_it_proves,
|
||||||
)
|
)
|
||||||
return Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
|
else:
|
||||||
|
res = Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
|
||||||
|
if c.after:
|
||||||
|
res.followups = c.after(raw)
|
||||||
|
return res
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if c.must_refuse and e.code in (401, 403, 503):
|
if c.must_refuse and e.code in (401, 403, 503):
|
||||||
return Result(c.name, "ok", f"correctly refused (HTTP {e.code})",
|
return Result(c.name, "ok", f"correctly refused (HTTP {e.code})",
|
||||||
@@ -110,6 +129,52 @@ def _probe(c: Check) -> Result:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def logout(token: str, *, attempts: int = LOGOUT_ATTEMPTS, gap: float = LOGOUT_GAP_S,
|
||||||
|
sleep: Callable[[float], None] = time.sleep) -> Result:
|
||||||
|
"""End the session the login probe opened. Honest: never ok unless proven."""
|
||||||
|
what = "the canary leaves no live session behind (journey cleanup rule)"
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "windy-git-canary/1.0",
|
||||||
|
"X-Windy-Synthetic": "1",
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
}
|
||||||
|
start = time.monotonic()
|
||||||
|
last = "no attempt"
|
||||||
|
for i in range(attempts):
|
||||||
|
if i:
|
||||||
|
sleep(gap)
|
||||||
|
req = urllib.request.Request(LOGOUT_URL, data=b"", method="POST", headers=headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||||
|
return Result("identity.logout", "ok", f"session ended (HTTP {r.status})",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code in LOGOUT_GONE:
|
||||||
|
return Result("identity.logout", "ok", f"session already over (HTTP {e.code})",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
if e.code < 500: # a 4xx won't change on retry: fail fast
|
||||||
|
return Result("identity.logout", "down", f"CLEANUP FAILED: HTTP {e.code}",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
last = f"HTTP {e.code}"
|
||||||
|
except Exception as e: # noqa: BLE001 — no response / timeout: retry
|
||||||
|
last = f"{type(e).__name__}"
|
||||||
|
return Result("identity.logout", "down",
|
||||||
|
f"CLEANUP FAILED after {attempts} tries: {last} (next run's logout heals it)",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
|
||||||
|
|
||||||
|
def _logout_after_login(raw: bytes) -> list[Result]:
|
||||||
|
try:
|
||||||
|
token = (json.loads(raw or b"{}") or {}).get("token")
|
||||||
|
except ValueError:
|
||||||
|
token = None
|
||||||
|
if not token:
|
||||||
|
return [Result("identity.logout", "down",
|
||||||
|
"CLEANUP FAILED: login returned no token to log out with",
|
||||||
|
0.0, "the canary leaves no live session behind (journey cleanup rule)")]
|
||||||
|
return [logout(token)]
|
||||||
|
|
||||||
|
|
||||||
def build_checks() -> list[Check]:
|
def build_checks() -> list[Check]:
|
||||||
checks = [
|
checks = [
|
||||||
Check(
|
Check(
|
||||||
@@ -187,6 +252,7 @@ def build_checks() -> list[Check]:
|
|||||||
method="POST",
|
method="POST",
|
||||||
body={"email": email, "password": pw},
|
body={"email": email, "password": pw},
|
||||||
warn_seconds=LOGIN_WARN_SECONDS,
|
warn_seconds=LOGIN_WARN_SECONDS,
|
||||||
|
after=_logout_after_login,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return checks
|
return checks
|
||||||
@@ -263,7 +329,10 @@ def main() -> int:
|
|||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
previous = load_state()
|
previous = load_state()
|
||||||
results = [_probe(c) for c in build_checks()]
|
results = []
|
||||||
|
for c in build_checks():
|
||||||
|
r = _probe(c)
|
||||||
|
results += [r, *r.followups]
|
||||||
|
|
||||||
print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n")
|
print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n")
|
||||||
for r in results:
|
for r in results:
|
||||||
|
|||||||
196
scripts/ci_hygiene.py
Normal file
196
scripts/ci_hygiene.py
Normal file
@@ -0,0 +1,196 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""CI hygiene guard: installs come from a lockfile, never "latest" (house rule 6).
|
||||||
|
|
||||||
|
A floating install lets CI test different versions than prod ships, and a
|
||||||
|
rebuild silently changes prod. Windy Cloud's OpenAPI test failed on exactly
|
||||||
|
that (fastapi 0.141.1 in CI vs 0.136.0 on the dev box) and all three Cloud
|
||||||
|
cells floated in prod. Also flags services that publish a HOST port: every
|
||||||
|
CI job shares one dind daemon, so two jobs publishing 5432 collide ("port is
|
||||||
|
already allocated", Windy Mind runs 147/176).
|
||||||
|
|
||||||
|
WARN-ONLY (`windy-git/ci-hygiene`, green + "⚠ WARN"); CI_HYGIENE_MODE=block
|
||||||
|
turns it red once the lanes report clean. Scans CI workflow files and
|
||||||
|
Dockerfiles only. PR heads: lines the PR adds. Default branch: every line.
|
||||||
|
|
||||||
|
OK (not flagged):
|
||||||
|
pip / uv pip install -r FILE (with or without --require-hashes), --no-deps,
|
||||||
|
exact pins (tool==1.2.3), pip/setuptools/wheel upgrades
|
||||||
|
uv sync --locked | --frozen npm ci
|
||||||
|
npm install pkg@1.2.3 (every package exact-pinned)
|
||||||
|
yarn install --frozen-lockfile / --immutable pnpm install --frozen-lockfile
|
||||||
|
Also flagged: `:latest` images (FROM / COPY --from / image: / docker://) and
|
||||||
|
`COPY uv.lock* ...`-style globs that build without the lock (Windy Mail #147).
|
||||||
|
Exceptions: ci/ci-hygiene-allow.yml, one reason per entry.
|
||||||
|
|
||||||
|
python3 scripts/ci_hygiene.py report [repo ...]
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shlex
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import compute_guard as cg # noqa: E402 (shared walker, cache and allow-list loader)
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
ALLOW_FILE = Path(os.environ.get("CI_HYGIENE_ALLOW", ROOT / "ci" / "ci-hygiene-allow.yml"))
|
||||||
|
MODE = os.environ.get("CI_HYGIENE_MODE", "warn")
|
||||||
|
|
||||||
|
# CI workflow files and Dockerfiles; never vendored copies.
|
||||||
|
INCLUDE = re.compile(r"(^|/)\.(github|gitea)/workflows/[^/]+\.ya?ml$|(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
||||||
|
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
||||||
|
PREFILTER = (r"pip3? install|pip install|uv sync|npm (install|i )|yarn install|pnpm install"
|
||||||
|
r"|^\s*-\s*['\"]?[0-9]+:[0-9]+|:latest|lock[^ ]*\*")
|
||||||
|
|
||||||
|
TOOLING = {"pip", "setuptools", "wheel"}
|
||||||
|
DOCKER_FILE = re.compile(r"(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
||||||
|
LATEST = re.compile(r"(?:^\s*FROM\s+(?:--platform=\S+\s+)?|--from=|image:\s*['\"]?|docker://)([\w./-]+):latest\b", re.I)
|
||||||
|
LOCKNAME = re.compile(r"(uv\.lock|poetry\.lock|package-lock\.json|pnpm-lock\.yaml|yarn\.lock|requirements[^ ]*\.(txt|lock))", re.I)
|
||||||
|
EXACT_PY = re.compile(r"^[A-Za-z0-9._-]+(\[[^\]]*\])?==[A-Za-z0-9.+!-]+$")
|
||||||
|
EXACT_NPM = re.compile(r"^(@[^/@]+/)?[^/@]+@\d+\.\d+\.\d+([-+][0-9A-Za-z.-]+)?$")
|
||||||
|
HOST_PORT = re.compile(r"^\s*-\s*['\"]?(\d{2,5}):(\d{2,5})['\"]?\s*(#.*)?$")
|
||||||
|
PIP_VALUE_FLAGS = {"-c", "--constraint", "-i", "--index-url", "--extra-index-url", "-f",
|
||||||
|
"--find-links", "--target", "-t", "--python", "--prefix", "--root", "--platform",
|
||||||
|
"--python-version", "--implementation", "--abi", "--only-binary", "--no-binary"}
|
||||||
|
|
||||||
|
|
||||||
|
def path_ok(path: str) -> bool:
|
||||||
|
return bool(INCLUDE.search(path)) and not NEVER.search(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _commands(text: str) -> list[list[str]]:
|
||||||
|
"""Split a shell line into simple commands (&&, ||, ;, |), tokenized."""
|
||||||
|
out = []
|
||||||
|
for part in re.split(r"&&|\|\||;|\|", text):
|
||||||
|
try:
|
||||||
|
toks = shlex.split(part, comments=True)
|
||||||
|
except ValueError:
|
||||||
|
toks = part.split()
|
||||||
|
# Dockerfile RUN prefix / sudo / env-prefixed assignments
|
||||||
|
while toks and (toks[0] in ("RUN", "sudo", "exec", "-", "run:", "command:")
|
||||||
|
or re.match(r"^[A-Z_][A-Z0-9_]*=", toks[0])):
|
||||||
|
toks = toks[1:]
|
||||||
|
if toks:
|
||||||
|
out.append(toks)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _pip_problem(args: list[str]) -> str | None:
|
||||||
|
if "-r" in args or "--requirement" in args or any(a.startswith("--requirement=") for a in args):
|
||||||
|
return None
|
||||||
|
if "--no-deps" in args:
|
||||||
|
return None
|
||||||
|
pkgs, skip = [], False
|
||||||
|
for a in args:
|
||||||
|
if skip:
|
||||||
|
skip = False
|
||||||
|
continue
|
||||||
|
if a in PIP_VALUE_FLAGS:
|
||||||
|
skip = True
|
||||||
|
continue
|
||||||
|
if a.startswith("-") and a not in ("-e", "--editable"):
|
||||||
|
continue
|
||||||
|
if a in ("-e", "--editable"):
|
||||||
|
continue
|
||||||
|
pkgs.append(a)
|
||||||
|
loose = [p for p in pkgs if not EXACT_PY.match(p) and p.split("[")[0].lower() not in TOOLING]
|
||||||
|
if loose:
|
||||||
|
return f"floating pip install: {' '.join(loose)[:40]}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
||||||
|
if cg.COMMENT.match(text):
|
||||||
|
return []
|
||||||
|
hits = []
|
||||||
|
if "/workflows/" in path and HOST_PORT.match(text):
|
||||||
|
hits.append(("host port", f"service publishes host port {HOST_PORT.match(text).group(1)} (shared dind)"))
|
||||||
|
return hits
|
||||||
|
# Windy Mail #147: a `:latest` build/tool image floats exactly like an
|
||||||
|
# unpinned package, and `COPY uv.lock* ./` builds WITHOUT the lock when it
|
||||||
|
# is missing instead of failing.
|
||||||
|
m = LATEST.search(text)
|
||||||
|
if m:
|
||||||
|
hits.append(("floating image", f"{m.group(1)}:latest"))
|
||||||
|
if DOCKER_FILE.search(path) and re.match(r"^\s*COPY\b", text, re.I):
|
||||||
|
globbed = [t for t in text.split() if "*" in t and LOCKNAME.search(t)]
|
||||||
|
if globbed:
|
||||||
|
hits.append(("optional lock", f"COPY {globbed[0]} (must fail if the lock is missing)"))
|
||||||
|
for toks in _commands(text):
|
||||||
|
low = [t.lower() for t in toks]
|
||||||
|
# pip install / python -m pip install / uv pip install
|
||||||
|
for i in range(len(low) - 1):
|
||||||
|
if os.path.basename(low[i]) in ("pip", "pip3") and low[i + 1] == "install":
|
||||||
|
prob = _pip_problem(toks[i + 2:])
|
||||||
|
if prob:
|
||||||
|
hits.append(("floating install", prob))
|
||||||
|
break
|
||||||
|
if low[:2] == ["uv", "sync"] and not ({"--locked", "--frozen"} & set(low)):
|
||||||
|
hits.append(("floating install", "uv sync without --locked/--frozen"))
|
||||||
|
if low[:1] == ["npm"] and len(low) > 1 and low[1] in ("install", "i", "add"):
|
||||||
|
pkgs = [t for t in toks[2:] if not t.startswith("-")]
|
||||||
|
if not pkgs or not all(EXACT_NPM.match(p) for p in pkgs):
|
||||||
|
hits.append(("floating install", f"npm {low[1]} {' '.join(pkgs)[:30]}".strip() + " (use npm ci)"))
|
||||||
|
if low[:2] == ["yarn", "install"] and not ({"--frozen-lockfile", "--immutable"} & set(low)):
|
||||||
|
hits.append(("floating install", "yarn install without --frozen-lockfile"))
|
||||||
|
if low[:2] == ["pnpm", "install"] and "--frozen-lockfile" not in low:
|
||||||
|
hits.append(("floating install", "pnpm install without --frozen-lockfile"))
|
||||||
|
return hits
|
||||||
|
|
||||||
|
|
||||||
|
def check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
||||||
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
||||||
|
return None
|
||||||
|
allow = cg.load_allow(ALLOW_FILE)
|
||||||
|
rules = hashlib.sha256((PREFILTER + INCLUDE.pattern + EXACT_PY.pattern + EXACT_NPM.pattern).encode()).hexdigest()[:8]
|
||||||
|
fp = cg._fingerprint(allow) + ":" + rules # hashlib, not hash(): hash() is per-process random
|
||||||
|
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
||||||
|
if is_default_head:
|
||||||
|
return cg.cached_scan(f"hyg-tree:{repo}:{sha}:{fp}",
|
||||||
|
lambda: cg.scan_tree(repo, bare, sha, allow, prefilter=PREFILTER, **kw))
|
||||||
|
return cg.cached_scan(f"hyg-pr:{repo}:{sha}:{fp}",
|
||||||
|
lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow, **kw))
|
||||||
|
|
||||||
|
|
||||||
|
def status_for(findings, whole_tree: bool):
|
||||||
|
scope = "in CI/Dockerfiles" if whole_tree else "added"
|
||||||
|
if not findings:
|
||||||
|
return "success", f"OK: no floating install or host-port service {scope}", None
|
||||||
|
f = findings[0]
|
||||||
|
n = len(findings)
|
||||||
|
state = "failure" if MODE == "block" else "success"
|
||||||
|
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
||||||
|
return state, f"{lead}: {n} CI hygiene issue{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
||||||
|
|
||||||
|
|
||||||
|
def report(repos: list[str]) -> int:
|
||||||
|
allow = cg.load_allow(ALLOW_FILE)
|
||||||
|
total = 0
|
||||||
|
for repo in repos:
|
||||||
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir():
|
||||||
|
print(f"## {repo}: no sync clone, skipped")
|
||||||
|
continue
|
||||||
|
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||||
|
sha = cg._git(bare, "rev-parse", head).strip()
|
||||||
|
fs = cg.scan_tree(repo, bare, sha, allow, line_fn=scan_line, path_ok=path_ok, prefilter=PREFILTER)
|
||||||
|
total += len(fs)
|
||||||
|
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} issue(s)")
|
||||||
|
for f in fs:
|
||||||
|
print(f" {f.path}:{f.line} [{f.kind}] {f.match}")
|
||||||
|
print(f"TOTAL {total}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
||||||
|
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
||||||
|
p.name.removesuffix(".git") for p in cg.WORK.glob("*.git"))
|
||||||
|
sys.exit(report(sys.argv[2:] or default))
|
||||||
|
sys.exit(__doc__)
|
||||||
292
scripts/compute_guard.py
Normal file
292
scripts/compute_guard.py
Normal file
@@ -0,0 +1,292 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Compute guard: Windy Mind is the ONLY door to AI compute (Grant, 2026-09-23).
|
||||||
|
|
||||||
|
Flags code that talks to an AI provider directly instead of through Windy Mind:
|
||||||
|
a provider API host, a provider SDK import or dependency, or a raw provider key
|
||||||
|
name. Direct calls skip Mind's metering, caps and live-model routing, and they
|
||||||
|
spend whichever key happens to be lying around (the audit found Grant's personal
|
||||||
|
Max OAuth token inside a platform container).
|
||||||
|
|
||||||
|
WARN-ONLY for now: the bridge posts `windy-git/compute-guard` as success with a
|
||||||
|
"⚠ WARN" description, so nothing turns red. `COMPUTE_GUARD_MODE=block` flips
|
||||||
|
findings to failure once the repos are clean (orchestrator's call).
|
||||||
|
|
||||||
|
- PR heads: only lines the PR ADDS (vs its merge-base with the default branch).
|
||||||
|
- Default-branch head: the whole tree (the baseline, and what `report` prints).
|
||||||
|
|
||||||
|
Exceptions live in ONE file, ci/compute-guard-allow.yml, each with a reason.
|
||||||
|
Tests, docs, lockfiles, vendored code and CI config are never scanned.
|
||||||
|
Reads the sync's bare GitHub clones on Veron (no docker exec: IO-stall lesson).
|
||||||
|
|
||||||
|
python3 scripts/compute_guard.py report [repo ...] # whole-tree findings on each default branch
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
ALLOW_FILE = Path(os.environ.get("COMPUTE_GUARD_ALLOW", ROOT / "ci" / "compute-guard-allow.yml"))
|
||||||
|
WORK = Path(os.environ.get("SYNC_WORK", "/srv/windygit/sync"))
|
||||||
|
CACHE = Path(os.environ.get("COMPUTE_GUARD_CACHE", "/var/lib/windy-git/compute-guard-cache.json"))
|
||||||
|
MODE = os.environ.get("COMPUTE_GUARD_MODE", "warn") # warn | block
|
||||||
|
|
||||||
|
HOSTS = [
|
||||||
|
"api.anthropic.com", "api.openai.com", "api.groq.com",
|
||||||
|
"generativelanguage.googleapis.com", "api.mistral.ai", "api.perplexity.ai",
|
||||||
|
"openrouter.ai", "api.together.xyz", "api.together.ai", "api.cerebras.ai",
|
||||||
|
"api.sambanova.ai", "api.deepseek.com", "api.x.ai", "api.cohere.ai",
|
||||||
|
"api.cohere.com", "api.fireworks.ai", "api.replicate.com",
|
||||||
|
"api-inference.huggingface.co",
|
||||||
|
]
|
||||||
|
KEYS = [
|
||||||
|
"ANTHROPIC_API_KEY", "ANTHROPIC_OAUTH_TOKEN", "ANTHROPIC_AUTH_TOKEN",
|
||||||
|
"OPENAI_API_KEY", "GROQ_API_KEY", "GEMINI_API_KEY", "GOOGLE_GENERATIVE_AI_API_KEY",
|
||||||
|
"GOOGLE_AI_API_KEY", "MISTRAL_API_KEY", "PERPLEXITY_API_KEY", "PPLX_API_KEY",
|
||||||
|
"OPENROUTER_API_KEY", "TOGETHER_API_KEY", "CEREBRAS_API_KEY", "SAMBANOVA_API_KEY",
|
||||||
|
"DEEPSEEK_API_KEY", "XAI_API_KEY", "COHERE_API_KEY", "FIREWORKS_API_KEY",
|
||||||
|
"REPLICATE_API_TOKEN",
|
||||||
|
]
|
||||||
|
PY_SDKS = r"anthropic|openai|groq|mistralai|cohere|google\.generativeai|google\.genai|together|cerebras|litellm"
|
||||||
|
JS_SDKS = (r"@anthropic-ai/sdk|openai|groq-sdk|@google/generative-ai|@google/genai|@mistralai/mistralai"
|
||||||
|
r"|cohere-ai|together-ai|@ai-sdk/(?:anthropic|openai|groq|google|mistral)")
|
||||||
|
|
||||||
|
RULES: list[tuple[str, re.Pattern]] = [
|
||||||
|
("provider host", re.compile("|".join(re.escape(h) for h in HOSTS))),
|
||||||
|
("provider key", re.compile(r"\b(?:" + "|".join(KEYS) + r")\b")),
|
||||||
|
("provider SDK", re.compile(rf"^\s*(?:from|import)\s+(?:{PY_SDKS})(?:\s|\.|$|,)")),
|
||||||
|
("provider SDK", re.compile(rf"""(?:from\s+|require\(\s*|import\(\s*)['"](?:{JS_SDKS})(?:/[^'"]*)?['"]""")),
|
||||||
|
# dependency manifests: package.json keys, requirements / pyproject lines
|
||||||
|
("provider SDK dep", re.compile(rf'''^\s*"(?:{JS_SDKS})"\s*:''')),
|
||||||
|
("provider SDK dep", re.compile(rf'''^\s*["']?(?:{PY_SDKS.replace(chr(92) + ".", "-")})(?:\[[^\]]*\])?\s*(?:[<>=~!]=?|["',]|$)''')),
|
||||||
|
]
|
||||||
|
DEP_FILES = re.compile(r"(^|/)(package\.json|requirements[^/]*\.txt|pyproject\.toml|setup\.cfg|Pipfile)$")
|
||||||
|
|
||||||
|
# Never scanned: tests, docs, lockfiles, vendored/built code, CI config.
|
||||||
|
SKIP = re.compile(
|
||||||
|
r"(^|/)(tests?|__tests__|spec|docs?|node_modules|vendor|dist|build|\.github|\.gitea)/"
|
||||||
|
r"|(^|/)(test_[^/]*|[^/]*_test\.py|conftest\.py|[^/]*\.(test|spec)\.[cm]?[jt]sx?)$"
|
||||||
|
r"|\.(md|mdx|rst|txt|lock|snap|svg|png|jpg|pdf)$"
|
||||||
|
r"|(^|/)(package-lock\.json|pnpm-lock\.yaml|yarn\.lock|uv\.lock|poetry\.lock|Cargo\.lock)$"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Finding:
|
||||||
|
path: str
|
||||||
|
line: int
|
||||||
|
kind: str
|
||||||
|
match: str
|
||||||
|
|
||||||
|
|
||||||
|
def load_allow(path: Path = ALLOW_FILE) -> list[dict]:
|
||||||
|
data = yaml.safe_load(path.read_text()) or {}
|
||||||
|
entries = data.get("allow") or []
|
||||||
|
for e in entries: # a reason per entry is the whole point of the file
|
||||||
|
if not (e.get("repo") and e.get("paths") and str(e.get("reason", "")).strip()):
|
||||||
|
raise ValueError(f"allow entry needs repo, paths and a reason: {e}")
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def allowed(repo: str, path: str, allow: list[dict]) -> bool:
|
||||||
|
for e in allow:
|
||||||
|
if e["repo"] == repo and any(fnmatch.fnmatch(path, g) for g in e["paths"]):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
COMMENT = re.compile(r"^\s*(?:#|//|/\*|\*|<!--)")
|
||||||
|
|
||||||
|
|
||||||
|
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
||||||
|
# A comment is not a call: "the ANTHROPIC_OAUTH_TOKEN setting was removed"
|
||||||
|
# (windy-search) must not count, nor a commented-out `# OPENAI_API_KEY=`.
|
||||||
|
if COMMENT.match(text):
|
||||||
|
return []
|
||||||
|
hits = []
|
||||||
|
for kind, rx in RULES:
|
||||||
|
if kind == "provider SDK dep" and not DEP_FILES.search(path):
|
||||||
|
continue
|
||||||
|
m = rx.search(text)
|
||||||
|
if m:
|
||||||
|
hits.append((kind, m.group(0).strip()[:60]))
|
||||||
|
return hits
|
||||||
|
|
||||||
|
|
||||||
|
def _git(bare: Path, *args: str) -> str:
|
||||||
|
return subprocess.run(
|
||||||
|
["git", "--git-dir", str(bare), *args],
|
||||||
|
capture_output=True, text=True, check=True, timeout=120,
|
||||||
|
).stdout
|
||||||
|
|
||||||
|
|
||||||
|
def _default_path_ok(path: str) -> bool:
|
||||||
|
return not SKIP.search(path)
|
||||||
|
|
||||||
|
|
||||||
|
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict], *, line_fn=None,
|
||||||
|
path_ok=None, prefilter: str | None = None) -> list[Finding]:
|
||||||
|
"""Every line in the tree at `sha` (default branch: the baseline)."""
|
||||||
|
# A cheap prefilter by git, then the real rules in Python.
|
||||||
|
# Other guards (ci_hygiene) reuse this walker with their own line rules.
|
||||||
|
line_fn = line_fn or scan_line
|
||||||
|
path_ok = path_ok or _default_path_ok
|
||||||
|
pre = prefilter or "|".join([re.escape(h) for h in HOSTS] + KEYS + [
|
||||||
|
"anthropic", "openai", "groq", "mistral", "generativeai", "genai", "cohere",
|
||||||
|
"together", "cerebras", "litellm"])
|
||||||
|
try:
|
||||||
|
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
||||||
|
except subprocess.CalledProcessError as e:
|
||||||
|
if e.returncode == 1: # no matches
|
||||||
|
return []
|
||||||
|
raise
|
||||||
|
found = []
|
||||||
|
for raw in out.splitlines():
|
||||||
|
# <sha>:<path>:<line>:<text>
|
||||||
|
try:
|
||||||
|
_, path, line, text = raw.split(":", 3)
|
||||||
|
except ValueError:
|
||||||
|
continue
|
||||||
|
if not path_ok(path) or allowed(repo, path, allow):
|
||||||
|
continue
|
||||||
|
for kind, match in line_fn(path, text):
|
||||||
|
found.append(Finding(path, int(line), kind, match))
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict], **kw) -> list[Finding]:
|
||||||
|
"""Only the lines a PR adds, vs its merge-base with the default branch."""
|
||||||
|
mb = _git(bare, "merge-base", base_ref, sha).strip()
|
||||||
|
diff = _git(bare, "diff", "-U0", "--no-color", "--no-ext-diff", mb, sha)
|
||||||
|
return parse_added(repo, diff, allow, **kw)
|
||||||
|
|
||||||
|
|
||||||
|
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_added(repo: str, diff: str, allow: list[dict], *, line_fn=None, path_ok=None) -> list[Finding]:
|
||||||
|
line_fn = line_fn or scan_line
|
||||||
|
path_ok = path_ok or _default_path_ok
|
||||||
|
found, path, line = [], None, 0
|
||||||
|
for raw in diff.splitlines():
|
||||||
|
if raw.startswith("+++ "):
|
||||||
|
p = raw[4:]
|
||||||
|
path = None if p == "/dev/null" else p[2:] if p.startswith("b/") else p
|
||||||
|
continue
|
||||||
|
m = HUNK.match(raw)
|
||||||
|
if m:
|
||||||
|
line = int(m.group(1))
|
||||||
|
continue
|
||||||
|
if path is None or raw.startswith("--- "):
|
||||||
|
continue
|
||||||
|
if raw.startswith("+"):
|
||||||
|
if path_ok(path) and not allowed(repo, path, allow):
|
||||||
|
for kind, match in line_fn(path, raw[1:]):
|
||||||
|
found.append(Finding(path, line, kind, match))
|
||||||
|
line += 1
|
||||||
|
return found
|
||||||
|
|
||||||
|
|
||||||
|
# ---- cache: a tree scan runs once per (repo, sha, rules+allow) --------------
|
||||||
|
def _fingerprint(allow: list[dict]) -> str:
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps([HOSTS, KEYS, PY_SDKS, JS_SDKS, SKIP.pattern, allow], sort_keys=True).encode()
|
||||||
|
).hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def cached_scan(key: str, fn) -> list[Finding]:
|
||||||
|
try:
|
||||||
|
cache = json.loads(CACHE.read_text())
|
||||||
|
except (OSError, ValueError):
|
||||||
|
cache = {}
|
||||||
|
if key in cache:
|
||||||
|
return [Finding(**f) for f in cache[key]]
|
||||||
|
result = fn()
|
||||||
|
cache[key] = [f.__dict__ for f in result]
|
||||||
|
if len(cache) > 2000: # keep it small: newest entries win
|
||||||
|
cache = dict(list(cache.items())[-1000:])
|
||||||
|
try:
|
||||||
|
CACHE.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
tmp = CACHE.with_suffix(".tmp")
|
||||||
|
tmp.write_text(json.dumps(cache))
|
||||||
|
tmp.replace(CACHE)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def fetched(bare: Path, sha: str) -> bool:
|
||||||
|
"""Is `sha` in the sync clone yet? The bridge learns PR / default heads from
|
||||||
|
GitHub's API AFTER the sync fetched, so a push in between is simply not here
|
||||||
|
until the next 5-min cycle. That is a race, not an error: skip quietly."""
|
||||||
|
try:
|
||||||
|
_git(bare, "cat-file", "-e", f"{sha}^{{commit}}")
|
||||||
|
return True
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> list[Finding] | None:
|
||||||
|
"""Findings for one commit, or None when the guard can't run (never a fake OK)."""
|
||||||
|
bare = WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir() or not fetched(bare, sha):
|
||||||
|
return None
|
||||||
|
allow = load_allow()
|
||||||
|
fp = _fingerprint(allow)
|
||||||
|
if is_default_head:
|
||||||
|
return cached_scan(f"tree:{repo}:{sha}:{fp}", lambda: scan_tree(repo, bare, sha, allow))
|
||||||
|
return cached_scan(
|
||||||
|
f"pr:{repo}:{sha}:{fp}",
|
||||||
|
lambda: scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def status_for(findings: list[Finding], whole_tree: bool) -> tuple[str, str, Finding | None]:
|
||||||
|
"""(state, description, first finding) for the GitHub commit status."""
|
||||||
|
scope = "in tree" if whole_tree else "added"
|
||||||
|
if not findings:
|
||||||
|
what = "no direct AI-provider use in tree" if whole_tree else "no direct AI-provider use added"
|
||||||
|
return "success", f"OK: {what} (Windy Mind is the only door)", None
|
||||||
|
f = findings[0]
|
||||||
|
n = len(findings)
|
||||||
|
state = "failure" if MODE == "block" else "success"
|
||||||
|
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
||||||
|
desc = f"{lead}: {n} direct AI-provider use{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"
|
||||||
|
return state, desc[:140], f
|
||||||
|
|
||||||
|
|
||||||
|
def report(repos: list[str]) -> int:
|
||||||
|
allow = load_allow()
|
||||||
|
total = 0
|
||||||
|
for repo in repos:
|
||||||
|
bare = WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir():
|
||||||
|
print(f"## {repo}: no sync clone, skipped")
|
||||||
|
continue
|
||||||
|
head = _git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||||
|
sha = _git(bare, "rev-parse", head).strip()
|
||||||
|
fs = scan_tree(repo, bare, sha, allow)
|
||||||
|
total += len(fs)
|
||||||
|
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} finding(s)")
|
||||||
|
for f in fs:
|
||||||
|
print(f" {f.path}:{f.line} [{f.kind}] {f.match}")
|
||||||
|
print(f"TOTAL {total}")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
||||||
|
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
||||||
|
p.name.removesuffix(".git") for p in WORK.glob("*.git"))
|
||||||
|
sys.exit(report(sys.argv[2:] or default))
|
||||||
|
sys.exit(__doc__)
|
||||||
124
scripts/guards_report.py
Normal file
124
scripts/guards_report.py
Normal file
@@ -0,0 +1,124 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Live status of the repo guards (compute-guard + ci-hygiene) as one markdown page.
|
||||||
|
|
||||||
|
Scans every bridged repo's DEFAULT branch with both guards and renders what is
|
||||||
|
left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml:
|
||||||
|
windy-pro's desktop app and its build jobs) are listed in their OWN section and
|
||||||
|
do not count against "ready to block": those are proposals for Grant, not a
|
||||||
|
lane's fix (orchestrator, 09-23).
|
||||||
|
|
||||||
|
sudo python3 scripts/guards_report.py > GUARDS_STATUS.md
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import fnmatch
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||||
|
import ci_hygiene as hy # noqa: E402
|
||||||
|
import compute_guard as cg # noqa: E402
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml"))
|
||||||
|
REPOS = os.environ.get("BRIDGE_REPOS", "").split() or [
|
||||||
|
"windy-chat", "windy-mail", "windy-calendar", "Windy-Clone", "WindyCloud", "windy-search",
|
||||||
|
"windy-connect", "windy-drops", "windy-code-web", "windy-code", "windy-traveler",
|
||||||
|
"windy-registry", "eternitas", "windy-translate", "windytranslate-site", "windytraveler-site",
|
||||||
|
"windy-hand", "windy-cloud-sites", "windy-cloud-domains", "windy-cloud-vps", "windytalk",
|
||||||
|
"windy-pro", "windy-mind", "windy-git"]
|
||||||
|
JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
|
||||||
|
|
||||||
|
|
||||||
|
def job_of(text: str, line: int) -> str | None:
|
||||||
|
"""The workflow job a line belongs to (2-space keys under `jobs:`)."""
|
||||||
|
in_jobs, job = False, None
|
||||||
|
for i, raw in enumerate(text.splitlines(), 1):
|
||||||
|
if raw.startswith("jobs:"):
|
||||||
|
in_jobs = True
|
||||||
|
elif in_jobs and JOB.match(raw):
|
||||||
|
job = JOB.match(raw).group(1)
|
||||||
|
elif raw and not raw[0].isspace() and not raw.startswith("jobs:"):
|
||||||
|
in_jobs = False
|
||||||
|
if i == line:
|
||||||
|
return job if in_jobs else None
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def grant_owned(repo: str, path: str, job: str | None, owned: list[dict]) -> bool:
|
||||||
|
for e in owned:
|
||||||
|
if e["repo"] != repo:
|
||||||
|
continue
|
||||||
|
if any(fnmatch.fnmatch(path, g) for g in e.get("paths") or []):
|
||||||
|
return True
|
||||||
|
if job and job in (e.get("jobs") or {}).get(path, []):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def scan(repo: str, owned: list[dict]):
|
||||||
|
bare = cg.WORK / f"{repo}.git"
|
||||||
|
if not bare.is_dir():
|
||||||
|
return None
|
||||||
|
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||||
|
sha = cg._git(bare, "rev-parse", head).strip()
|
||||||
|
out = {"sha": sha, "compute": [], "hygiene": []}
|
||||||
|
texts: dict[str, str] = {}
|
||||||
|
for key, fs in (("compute", cg.check(repo, sha, head, True) or []),
|
||||||
|
("hygiene", hy.check(repo, sha, head, True) or [])):
|
||||||
|
for f in fs:
|
||||||
|
job = None
|
||||||
|
if "/workflows/" in f.path:
|
||||||
|
if f.path not in texts:
|
||||||
|
texts[f.path] = cg._git(bare, "show", f"{sha}:{f.path}")
|
||||||
|
job = job_of(texts[f.path], f.line)
|
||||||
|
out[key].append((f, job, grant_owned(repo, f.path, job, owned)))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def render(results: dict) -> str:
|
||||||
|
now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime())
|
||||||
|
lane = {k: 0 for k in ("compute", "hygiene")}
|
||||||
|
grant = {k: 0 for k in ("compute", "hygiene")}
|
||||||
|
for r in results.values():
|
||||||
|
for k in lane:
|
||||||
|
lane[k] += sum(1 for _, _, g in r[k] if not g)
|
||||||
|
grant[k] += sum(1 for _, _, g in r[k] if g)
|
||||||
|
L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)",
|
||||||
|
"_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. "
|
||||||
|
"Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "",
|
||||||
|
"| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|",
|
||||||
|
f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |",
|
||||||
|
f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |",
|
||||||
|
"", "## By repo (lane-owned)", "| Repo | head | compute | hygiene | first items |", "|---|---|---|---|---|"]
|
||||||
|
for repo, r in sorted(results.items()):
|
||||||
|
c = [x for x in r["compute"] if not x[2]]
|
||||||
|
h = [x for x in r["hygiene"] if not x[2]]
|
||||||
|
items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (c + h)[:3]) or "clean ✅"
|
||||||
|
L.append(f"| {repo} | {r['sha'][:7]} | {len(c)} | {len(h)} | {items} |")
|
||||||
|
L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"]
|
||||||
|
g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene")
|
||||||
|
for f, job, own in r[k] if own]
|
||||||
|
L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"]
|
||||||
|
return "\n".join(L) + "\n"
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
owned = (yaml.safe_load(OWNED.read_text()) or {}).get("grant_owned") or []
|
||||||
|
results = {}
|
||||||
|
for repo in REPOS:
|
||||||
|
r = scan(repo, owned)
|
||||||
|
if r is not None:
|
||||||
|
results[repo] = r
|
||||||
|
sys.stdout.write(render(results))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
@@ -32,6 +32,7 @@ import base64
|
|||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
@@ -222,6 +223,54 @@ def sync_prs(repo: str) -> list[str]:
|
|||||||
return heads
|
return heads
|
||||||
|
|
||||||
|
|
||||||
|
SAFE_NAME = re.compile(r"^[A-Za-z0-9._-]+$")
|
||||||
|
SAFE_SHA = re.compile(r"^[0-9a-f]{40}$")
|
||||||
|
|
||||||
|
|
||||||
|
def queued_jobs(repo: str, sha: str) -> list[dict]:
|
||||||
|
"""Jobs at `sha` that are waiting for a runner and that a runner CAN take.
|
||||||
|
|
||||||
|
Gitea 1.24's API lists only PICKED-UP jobs (/actions/tasks), so a queued PR
|
||||||
|
showed nothing on GitHub and people asked whether the push was lost. The
|
||||||
|
truth is in the gitea DB. Bounded + non-fatal: during the 09-23 IO stall
|
||||||
|
`docker exec` hung for an hour and must never wedge the bridge again.
|
||||||
|
|
||||||
|
Only status 5 (waiting) with labels some live runner has. A `pending` we
|
||||||
|
post must end in a verdict we will also see, or it sits yellow forever:
|
||||||
|
blocked jobs (7) often end SKIPPED, and jobs for labels no runner has
|
||||||
|
(macos-/windows-/ubuntu-latest) are cancelled by the janitor unpicked;
|
||||||
|
neither ever appears in /actions/tasks.
|
||||||
|
"""
|
||||||
|
if not (SAFE_NAME.match(repo) and SAFE_NAME.match(WG_OWNER) and SAFE_SHA.match(sha)):
|
||||||
|
return []
|
||||||
|
query = (
|
||||||
|
"select json_build_object("
|
||||||
|
" 'jobs', (select coalesce(json_agg(t), '[]'::json) from ("
|
||||||
|
" select ar.index as run_number, ar.workflow_id, j.name, j.runs_on"
|
||||||
|
" from action_run_job j join action_run ar on ar.id = j.run_id"
|
||||||
|
" join repository r on r.id = j.repo_id join \"user\" o on o.id = r.owner_id"
|
||||||
|
f" where o.lower_name = '{WG_OWNER.lower()}' and r.lower_name = '{repo.lower()}'"
|
||||||
|
f" and ar.commit_sha = '{sha}' and j.status = 5) t),"
|
||||||
|
" 'labels', (select coalesce(json_agg(agent_labels), '[]'::json)"
|
||||||
|
" from action_runner where coalesce(deleted, 0) = 0));"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
out = subprocess.run(
|
||||||
|
["docker", "exec", "-i", "windy-git-db-1", "sh", "-c",
|
||||||
|
'psql -U "$POSTGRES_USER" -d gitea -At -v ON_ERROR_STOP=1'],
|
||||||
|
input=query, capture_output=True, text=True, check=True, timeout=30,
|
||||||
|
).stdout.strip()
|
||||||
|
got = json.loads(out or "{}")
|
||||||
|
runners = [set(json.loads(x or "[]")) for x in got.get("labels") or []]
|
||||||
|
return [
|
||||||
|
j for j in got.get("jobs") or []
|
||||||
|
if any(set(json.loads(j.get("runs_on") or "[]")) <= r for r in runners)
|
||||||
|
]
|
||||||
|
except (subprocess.SubprocessError, OSError, ValueError) as e:
|
||||||
|
print(f" {repo}: queued-job lookup skipped ({type(e).__name__})")
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
def post_statuses(repo: str, sha: str) -> None:
|
def post_statuses(repo: str, sha: str) -> None:
|
||||||
# Gitea caps a page at 50 (MAX_RESPONSE_ITEMS) whatever `limit` says, and a
|
# Gitea caps a page at 50 (MAX_RESPONSE_ITEMS) whatever `limit` says, and a
|
||||||
# daily scheduled workflow can push a quiet main's runs off page 1.
|
# daily scheduled workflow can push a quiet main's runs off page 1.
|
||||||
@@ -242,6 +291,17 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
ctx = f"windy-git/{r['workflow_id'].removesuffix('.yml')}/{r['name']}"
|
ctx = f"windy-git/{r['workflow_id'].removesuffix('.yml')}/{r['name']}"
|
||||||
if ctx not in latest or r["id"] > latest[ctx]["id"]:
|
if ctx not in latest or r["id"] > latest[ctx]["id"]:
|
||||||
latest[ctx] = r
|
latest[ctx] = r
|
||||||
|
# Queued jobs: `pending` where nothing newer has been picked up. A re-run
|
||||||
|
# queued behind an old failure must read pending, not the stale red.
|
||||||
|
for q in queued_jobs(repo, sha):
|
||||||
|
if NO_DAEMON_JOB.search(q["name"]):
|
||||||
|
continue
|
||||||
|
wf = q["workflow_id"].removesuffix(".yml")
|
||||||
|
if f"{wf}/{q['name']}" in NON_BLOCKING.get(repo, ()):
|
||||||
|
continue
|
||||||
|
ctx = f"windy-git/{wf}/{q['name']}"
|
||||||
|
if ctx not in latest or q["run_number"] > latest[ctx]["run_number"]:
|
||||||
|
latest[ctx] = {"id": 0, "status": "waiting", "run_number": q["run_number"]}
|
||||||
bad = invalid_workflows(repo, sha)
|
bad = invalid_workflows(repo, sha)
|
||||||
if not (latest or bad):
|
if not (latest or bad):
|
||||||
return
|
return
|
||||||
@@ -283,6 +343,48 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
print(f" {repo}@{sha[:7]} {ctx} = {state} -> {st}")
|
print(f" {repo}@{sha[:7]} {ctx} = {state} -> {st}")
|
||||||
|
|
||||||
|
|
||||||
|
GUARD_CTX = "windy-git/compute-guard"
|
||||||
|
HYGIENE_CTX = "windy-git/ci-hygiene"
|
||||||
|
|
||||||
|
|
||||||
|
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
|
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only)."""
|
||||||
|
_post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head)
|
||||||
|
|
||||||
|
|
||||||
|
def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
|
"""House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only)."""
|
||||||
|
_post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head)
|
||||||
|
|
||||||
|
|
||||||
|
def _post_guard(modname: str, ctx: str, repo: str, sha: str, default_branch: str,
|
||||||
|
is_default_head: bool) -> None:
|
||||||
|
"""One code path for every repo-scanning guard. Non-fatal and never a fake OK:
|
||||||
|
if the guard can't run, nothing is posted."""
|
||||||
|
try:
|
||||||
|
import importlib
|
||||||
|
|
||||||
|
g = importlib.import_module(modname) # same directory; lazy so the bridge never depends on it
|
||||||
|
findings = g.check(repo, sha, default_branch, is_default_head)
|
||||||
|
except Exception as e: # noqa: BLE001 — a guard must never break CI signals
|
||||||
|
print(f" {repo}@{sha[:7]} {ctx} skipped ({type(e).__name__}: {str(e)[:80]})")
|
||||||
|
return
|
||||||
|
if findings is None:
|
||||||
|
return
|
||||||
|
state, desc, first = g.status_for(findings, whole_tree=is_default_head)
|
||||||
|
st, existing = github("GET", f"/repos/{GH_OWNER}/{repo}/commits/{sha}/statuses?per_page=100")
|
||||||
|
for s in existing or []: # newest first: compare the latest guard status only
|
||||||
|
if s["context"] == ctx:
|
||||||
|
if (s["state"], s.get("description")) == (state, desc):
|
||||||
|
return
|
||||||
|
break
|
||||||
|
url = (f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}/{first.path}#L{first.line}"
|
||||||
|
if first else f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}")
|
||||||
|
st, _ = github("POST", f"/repos/{GH_OWNER}/{repo}/statuses/{sha}",
|
||||||
|
{"state": state, "context": ctx, "description": desc, "target_url": url})
|
||||||
|
print(f" {repo}@{sha[:7]} {ctx} = {state} ({len(findings)} finding(s)) -> {st}")
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
if not (GITEA_TOKEN and GITHUB_TOKEN):
|
if not (GITEA_TOKEN and GITHUB_TOKEN):
|
||||||
sys.exit("GITEA_ADMIN_TOKEN and GITHUB_TOKEN are required")
|
sys.exit("GITEA_ADMIN_TOKEN and GITHUB_TOKEN are required")
|
||||||
@@ -290,13 +392,18 @@ def main() -> int:
|
|||||||
for repo in REPOS:
|
for repo in REPOS:
|
||||||
try:
|
try:
|
||||||
shas = sync_prs(repo)
|
shas = sync_prs(repo)
|
||||||
|
default_branch, default_head = "main", None
|
||||||
st, br = github("GET", f"/repos/{GH_OWNER}/{repo}")
|
st, br = github("GET", f"/repos/{GH_OWNER}/{repo}")
|
||||||
if st == 200:
|
if st == 200:
|
||||||
st, b = github("GET", f"/repos/{GH_OWNER}/{repo}/branches/{br['default_branch']}")
|
default_branch = br["default_branch"]
|
||||||
|
st, b = github("GET", f"/repos/{GH_OWNER}/{repo}/branches/{default_branch}")
|
||||||
if st == 200:
|
if st == 200:
|
||||||
shas.append(b["commit"]["sha"])
|
default_head = b["commit"]["sha"]
|
||||||
|
shas.append(default_head)
|
||||||
for sha in dict.fromkeys(shas):
|
for sha in dict.fromkeys(shas):
|
||||||
post_statuses(repo, sha)
|
post_statuses(repo, sha)
|
||||||
|
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
||||||
|
post_ci_hygiene(repo, sha, default_branch, sha == default_head)
|
||||||
except Exception as e: # one repo's failure must not hide the others'
|
except Exception as e: # one repo's failure must not hide the others'
|
||||||
print(f" FAILED {repo}: {e}")
|
print(f" FAILED {repo}: {e}")
|
||||||
failed = 1
|
failed = 1
|
||||||
|
|||||||
49
scripts/rerun_ci.sh
Executable file
49
scripts/rerun_ci.sh
Executable file
@@ -0,0 +1,49 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Re-run a PR's (or branch's) CI on Windy Git. Runs ON Veron 1.
|
||||||
|
#
|
||||||
|
# bash scripts/rerun_ci.sh <repo> <branch> <sha-prefix>
|
||||||
|
#
|
||||||
|
# Gitea 1.24 has NO rerun API; the web button needs a hub-SSO session as
|
||||||
|
# windyadmin, which is Grant's identity, so we don't use it. Instead: move the
|
||||||
|
# Windy Git branch back one commit, let the next sync force-push the GitHub head
|
||||||
|
# again, and Gitea fires an ordinary push / pull_request_sync event on the SAME
|
||||||
|
# commit. Every workflow on that event re-runs, not only the failed one.
|
||||||
|
#
|
||||||
|
# Safety: refuses unless the branch is exactly at <sha-prefix> (GitHub's head),
|
||||||
|
# never rewinds while a sync is running (a run already past this repo would
|
||||||
|
# not push it back), waits for a sync that STARTS after the rewind, and if the
|
||||||
|
# branch is not verifiably back at <sha-prefix> by the deadline, restores it
|
||||||
|
# itself, so Windy Git is never left behind GitHub.
|
||||||
|
set -euo pipefail
|
||||||
|
repo="${1:?repo}"; branch="${2:?branch}"; want="${3:?sha prefix}"
|
||||||
|
G="sudo docker exec -u git windy-git-gitea-1 git -C /data/git/repositories/windyadmin/${repo}.git"
|
||||||
|
|
||||||
|
head=$($G rev-parse "refs/heads/${branch}")
|
||||||
|
[[ "$head" == "$want"* ]] || { echo "refusing: ${branch} is at ${head:0:7}, not ${want}"; exit 1; }
|
||||||
|
parent=$($G rev-parse "${head}^")
|
||||||
|
|
||||||
|
# NOT `systemctl is-active`: the sync is Type=oneshot, which reads "activating"
|
||||||
|
# (exit 3) for its whole run, so is-active says "idle" mid-run.
|
||||||
|
busy() { case "$(systemctl show windygit-sync -p ActiveState --value)" in
|
||||||
|
activating|active|deactivating|reloading) return 0;; esac; return 1; }
|
||||||
|
while busy; do sleep 5; done
|
||||||
|
$G update-ref "refs/heads/${branch}" "$parent" "$head"
|
||||||
|
mark=$(awk '{print int($1*1000000)}' /proc/uptime)
|
||||||
|
echo "rewound ${repo}:${branch} ${head:0:7} -> ${parent:0:7}"
|
||||||
|
|
||||||
|
deadline=$(( $(date +%s) + 900 ))
|
||||||
|
until [ "$(systemctl show windygit-sync -p ExecMainStartTimestampMonotonic --value)" -gt "$mark" ] \
|
||||||
|
&& [ "$($G rev-parse "refs/heads/${branch}")" = "$head" ]; do
|
||||||
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||||
|
$G update-ref "refs/heads/${branch}" "$head" "$($G rev-parse "refs/heads/${branch}")" || true
|
||||||
|
echo "TIMEOUT: restored ${branch} to ${head:0:7} by hand; NO new run fired"; exit 1
|
||||||
|
fi
|
||||||
|
sleep 10
|
||||||
|
done
|
||||||
|
echo "restored by sync: ${branch} = ${head:0:7}"
|
||||||
|
sleep 5
|
||||||
|
~/bin/wg-q <<SQL
|
||||||
|
select ar.index, ar.workflow_id, ar.event, ar.status, to_char(to_timestamp(ar.created),'HH24:MI:SS')
|
||||||
|
from action_run ar join repository r on r.id = ar.repo_id
|
||||||
|
where r.name = '${repo}' and ar.commit_sha = '${head}' order by ar.id desc limit 6;
|
||||||
|
SQL
|
||||||
@@ -82,7 +82,11 @@ done
|
|||||||
|
|
||||||
# Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait
|
# Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait
|
||||||
# forever and invisibly; cancel them after 30 min. Never fails the sync.
|
# forever and invisibly; cancel them after 30 min. Never fails the sync.
|
||||||
bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed (non-fatal)"
|
# Both DB steps go through `docker exec`, which hangs outright while the host
|
||||||
|
# is in an IO stall (09-23: data2 SMR cliff wedged this sync for 10+ min and
|
||||||
|
# stopped mirroring + the bridge for every lane). They are optional; mirroring
|
||||||
|
# and the bridge are not. Bound them so a stuck exec costs one step, not the run.
|
||||||
|
timeout -k 10 120 bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed or timed out (non-fatal)"
|
||||||
|
|
||||||
# Private repos can't run GitHub Actions; mirror their open PRs here so CI
|
# Private repos can't run GitHub Actions; mirror their open PRs here so CI
|
||||||
# fires, and post the verdicts back to GitHub as commit statuses.
|
# fires, and post the verdicts back to GitHub as commit statuses.
|
||||||
@@ -92,7 +96,7 @@ fi
|
|||||||
|
|
||||||
# CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
|
# CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
|
||||||
# Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
|
# Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
|
||||||
python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed (non-fatal)"
|
timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)"
|
||||||
|
|
||||||
[[ "$FAILED" -ne 0 ]] && { log "COMPLETED WITH FAILURES"; exit 1; }
|
[[ "$FAILED" -ne 0 ]] && { log "COMPLETED WITH FAILURES"; exit 1; }
|
||||||
log "all repos in step with GitHub"
|
log "all repos in step with GitHub"
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
|
import re
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
import time
|
import time
|
||||||
@@ -69,6 +70,100 @@ def iso(epoch: float) -> str:
|
|||||||
return datetime.fromtimestamp(epoch, UTC).isoformat().replace("+00:00", "Z")
|
return datetime.fromtimestamp(epoch, UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
|
||||||
|
|
||||||
|
# ---- push velocity: DETECT + ALERT ONLY (G3.4, 2026-09-23) -----------------
|
||||||
|
# `git push` goes straight to Gitea and never touches our API, so throttle.py
|
||||||
|
# cannot see it (NOT_ENFORCED_HERE). Gitea's own `action` table does record
|
||||||
|
# every push, so we read it here, emit `forge.push_velocity` when an account
|
||||||
|
# crosses a threshold, and let Telemetry Boss's detector page. Nothing here sits
|
||||||
|
# in the push path; nothing is ever refused (orchestrator, 09-23).
|
||||||
|
#
|
||||||
|
# Thresholds = the STANDARD-band bases from config.py (500 pushes/day; the
|
||||||
|
# force-push base of 10/day is used for ref deletes, the closest thing we can
|
||||||
|
# see). EI band multipliers are NOT applied: a platinum agent over 500/day is
|
||||||
|
# still flagged, for a human to look at, not blocked. Gitea records no
|
||||||
|
# "forced" flag, so force pushes cannot be told apart from pushes: named, not
|
||||||
|
# guessed.
|
||||||
|
PV_RULES = ( # (rule, row key, window_s, threshold)
|
||||||
|
("pushes_1h", "p1h", 3600, 60),
|
||||||
|
("pushes_24h", "p24h", 86400, 500),
|
||||||
|
("ref_deletes_24h", "d24h", 86400, 10),
|
||||||
|
)
|
||||||
|
# The GitHub -> Windy Git sync pushes as windyadmin every 5 min, by design.
|
||||||
|
PV_EXEMPT = {"windyadmin"}
|
||||||
|
# Gitea op_type: 5 commit push, 9 tag push, 16 tag delete, 17 branch delete.
|
||||||
|
# One action row per WATCHER is written for each push; user_id = act_user_id
|
||||||
|
# keeps exactly the actor's own copy.
|
||||||
|
PV_QUERY = """
|
||||||
|
select a.act_user_id as uid, u.lower_name as login,
|
||||||
|
(select el.external_id from external_login_user el
|
||||||
|
where el.user_id = a.act_user_id order by el.external_id limit 1) as wid,
|
||||||
|
count(*) filter (where a.op_type in (5, 9) and a.created_unix > {h1}) as p1h,
|
||||||
|
count(*) filter (where a.op_type in (5, 9)) as p24h,
|
||||||
|
count(*) filter (where a.op_type in (16, 17)) as d24h,
|
||||||
|
count(distinct a.repo_id) as repos
|
||||||
|
from action a join "user" u on u.id = a.act_user_id
|
||||||
|
where a.created_unix > {h24} and a.user_id = a.act_user_id
|
||||||
|
and a.op_type in (5, 9, 16, 17)
|
||||||
|
group by 1, 2"""
|
||||||
|
|
||||||
|
|
||||||
|
def passport_from_login(login: str) -> str | None:
|
||||||
|
"""agent-et26abcd1234 -> ET26-ABCD-1234 (repos.py _owner_login, reversed)."""
|
||||||
|
m = re.fullmatch(r"agent-([a-z0-9]{4})([a-z0-9]{4})([a-z0-9]{4})", login)
|
||||||
|
return "-".join(g.upper() for g in m.groups()) if m else None
|
||||||
|
|
||||||
|
|
||||||
|
def push_velocity_events(rows: list[dict], now: float, alerted: dict) -> tuple[list[dict], dict]:
|
||||||
|
"""(events, alerted') — one row per account per rule per window while over.
|
||||||
|
|
||||||
|
`alerted` maps "<uid>:<rule>" -> epoch of the last row. An account still over
|
||||||
|
the line is re-reported once per window, not every 5 minutes; one that drops
|
||||||
|
back under is forgotten, so a later burst reports again.
|
||||||
|
"""
|
||||||
|
events, keep = [], {}
|
||||||
|
for r in rows:
|
||||||
|
login = str(r["login"])
|
||||||
|
if login in PV_EXEMPT:
|
||||||
|
continue
|
||||||
|
agent = login.startswith("agent-")
|
||||||
|
for rule, key, window, limit in PV_RULES:
|
||||||
|
n = int(r[key])
|
||||||
|
if n <= limit:
|
||||||
|
continue
|
||||||
|
k = f"{r['uid']}:{rule}"
|
||||||
|
last = alerted.get(k)
|
||||||
|
if last is not None and now - float(last) < window:
|
||||||
|
keep[k] = last
|
||||||
|
continue
|
||||||
|
keep[k] = now
|
||||||
|
ev = {
|
||||||
|
"ts": iso(now),
|
||||||
|
"platform": PLATFORM,
|
||||||
|
"service": "forge",
|
||||||
|
"event_type": "forge.push_velocity",
|
||||||
|
"metadata": {
|
||||||
|
"rule": rule,
|
||||||
|
"window_s": window,
|
||||||
|
"count": n,
|
||||||
|
"threshold": limit,
|
||||||
|
"repos": int(r["repos"]),
|
||||||
|
"gitea_user_id": int(r["uid"]),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
# Actor rule (telemetry UPDATE 2): agent/human rows MUST carry an
|
||||||
|
# actor_id. Humans sign in to the forge only via Windy SSO, so the
|
||||||
|
# external login id IS their windy_identity_id. No id we can prove
|
||||||
|
# -> actor_type system + metadata.caller, never an invented id (I-12).
|
||||||
|
actor_id = passport_from_login(login) if agent else (r.get("wid") or None)
|
||||||
|
if actor_id:
|
||||||
|
ev["actor_type"], ev["actor_id"] = ("agent" if agent else "human"), str(actor_id)
|
||||||
|
else:
|
||||||
|
ev["actor_type"] = "system"
|
||||||
|
ev["metadata"]["caller"] = "unknown"
|
||||||
|
events.append(ev)
|
||||||
|
return events, keep
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
dry = "--dry-run" in sys.argv
|
dry = "--dry-run" in sys.argv
|
||||||
state = load_state()
|
state = load_state()
|
||||||
@@ -182,6 +277,20 @@ def main() -> int:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Isolated: a failing push-velocity query must never cost the ci.run rows.
|
||||||
|
pv_alerted = state.get("pv_alerted", {})
|
||||||
|
try:
|
||||||
|
pv_rows = sql(PV_QUERY.format(h1=int(now) - 3600, h24=int(now) - 86400))
|
||||||
|
pv_events, pv_alerted = push_velocity_events(pv_rows, now, pv_alerted)
|
||||||
|
except (subprocess.CalledProcessError, ValueError, KeyError) as e:
|
||||||
|
print(f"[telemetry] push velocity check FAILED (non-fatal): {type(e).__name__}")
|
||||||
|
pv_events = []
|
||||||
|
for e in pv_events:
|
||||||
|
m = e["metadata"]
|
||||||
|
print(f"[telemetry] WARNING push velocity: gitea user {m['gitea_user_id']} "
|
||||||
|
f"{m['rule']} = {m['count']} > {m['threshold']}")
|
||||||
|
events += pv_events
|
||||||
|
|
||||||
# ci.job_cancelled: spooled by the janitor (cancel_unrunnable.sh), one JSON per job.
|
# ci.job_cancelled: spooled by the janitor (cancel_unrunnable.sh), one JSON per job.
|
||||||
spool = os.environ.get("JANITOR_SPOOL", "/var/lib/windy-git/janitor-cancelled.jsonl")
|
spool = os.environ.get("JANITOR_SPOOL", "/var/lib/windy-git/janitor-cancelled.jsonl")
|
||||||
spooled = 0
|
spooled = 0
|
||||||
@@ -264,7 +373,7 @@ def main() -> int:
|
|||||||
with open(STATE + ".tmp", "w") as f:
|
with open(STATE + ".tmp", "w") as f:
|
||||||
json.dump(
|
json.dump(
|
||||||
{"last_fin": new_fin, "last_id": new_id, "last_ts": now,
|
{"last_fin": new_fin, "last_id": new_id, "last_ts": now,
|
||||||
"quarantined_unreported": quarantined},
|
"quarantined_unreported": quarantined, "pv_alerted": pv_alerted},
|
||||||
f,
|
f,
|
||||||
)
|
)
|
||||||
os.replace(STATE + ".tmp", STATE)
|
os.replace(STATE + ".tmp", STATE)
|
||||||
|
|||||||
Reference in New Issue
Block a user