Compare commits
4 Commits
ci-inputs-
...
87dcddb87d
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
87dcddb87d | ||
|
|
a5f7b036a8 | ||
|
|
255aa58b35 | ||
|
|
e64a1b5fcb |
@@ -51,9 +51,10 @@ jobs:
|
||||
- name: install
|
||||
run: |
|
||||
python3 --version
|
||||
python3 -m venv .venv
|
||||
.venv/bin/pip install -q --upgrade pip
|
||||
.venv/bin/pip install -e ".[dev]"
|
||||
# From uv.lock, never floating: CI tests exactly what the image ships.
|
||||
# --locked also FAILS if pyproject.toml changed without re-locking.
|
||||
python3 -m pip install -q uv==0.12.5
|
||||
uv sync --locked --extra dev
|
||||
|
||||
- name: lint
|
||||
run: .venv/bin/ruff check api scripts
|
||||
|
||||
15
Dockerfile
15
Dockerfile
@@ -10,10 +10,19 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends git curl \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY pyproject.toml ./
|
||||
RUN pip install --no-cache-dir -e .
|
||||
|
||||
# Dependencies come from uv.lock, hash-pinned, never "latest at build time".
|
||||
# Floating installs meant a rebuild could ship different fastapi/starlette/
|
||||
# pydantic than CI tested (Windy Cloud's OpenAPI drift, 09-23). The lock was
|
||||
# cut to exactly what prod ran then. uv only exports; pip installs, so the
|
||||
# image layout (system python, uvicorn on PATH) is unchanged.
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv
|
||||
COPY pyproject.toml uv.lock ./
|
||||
RUN uv export --frozen --no-dev --no-emit-project -o /tmp/requirements.txt \
|
||||
&& pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \
|
||||
&& rm /tmp/requirements.txt
|
||||
COPY api ./api
|
||||
RUN pip install --no-cache-dir --no-deps -e .
|
||||
|
||||
COPY alembic ./alembic
|
||||
COPY alembic.ini ./
|
||||
COPY scripts ./scripts
|
||||
|
||||
118
api/tests/test_ci_hygiene.py
Normal file
118
api/tests/test_ci_hygiene.py
Normal file
@@ -0,0 +1,118 @@
|
||||
"""CI hygiene guard (house rule 6): lockfile-only installs, no host-port services."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
_spec = importlib.util.spec_from_file_location("ci_hygiene", ROOT / "scripts" / "ci_hygiene.py")
|
||||
hy = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["ci_hygiene"] = hy
|
||||
_spec.loader.exec_module(hy)
|
||||
|
||||
WF = ".github/workflows/ci.yml"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, text", [
|
||||
(WF, " .venv/bin/pip install -e \".[dev]\""), # windy-git's own, before e64a1b5
|
||||
("Dockerfile", "RUN pip install --no-cache-dir -e ."), # windy-git image, before e64a1b5
|
||||
(WF, " - run: uv pip install -e \".[dev]\""), # WindyCloud #109's CI
|
||||
(WF, " run: pip install fastapi uvicorn"),
|
||||
(WF, " - run: uv sync --all-extras"), # windy-mind style, not locked
|
||||
(WF, " - run: npm install"), # windy-drops / windytalk
|
||||
(WF, " - run: npm install --no-save --no-audit --no-fund jsdom"), # windy-pro reality-check
|
||||
(WF, " - run: yarn install"),
|
||||
(WF, " - run: cd web && pnpm install"),
|
||||
("docker/api.Dockerfile", "RUN apt-get update && pip install requests"),
|
||||
])
|
||||
def test_floating_installs_are_flagged(path, text):
|
||||
assert [k for k, _ in hy.scan_line(path, text)] == ["floating install"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, text", [
|
||||
(WF, " python3 -m pip install -q uv==0.12.5"), # exact tool pin
|
||||
(WF, " uv sync --locked --extra dev"),
|
||||
(WF, " - run: uv sync --frozen"),
|
||||
(WF, " - run: npm ci"),
|
||||
(WF, " - run: npm install --no-save jsdom@24.1.0"),
|
||||
(WF, " - run: pip install -r requirements.lock --require-hashes"),
|
||||
(WF, " - run: pip install -r requirements.txt"),
|
||||
("Dockerfile", " && pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \\\\"),
|
||||
("Dockerfile", "RUN pip install --no-cache-dir --no-deps -e ."), # project only, deps from the lock
|
||||
(WF, " .venv/bin/pip install -q --upgrade pip"),
|
||||
(WF, " - run: yarn install --frozen-lockfile"),
|
||||
(WF, " # - run: npm install (commented out)"),
|
||||
(WF, " - run: echo 'pip is great'"),
|
||||
])
|
||||
def test_locked_or_pinned_installs_pass(path, text):
|
||||
assert hy.scan_line(path, text) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text, port", [
|
||||
(" - 5432:5432", "5432"), # windy-mind / eternitas (collided 09-23)
|
||||
(" - '15432:5432'", "15432"), # WindyCloud
|
||||
(' - "6379:6379"', "6379"),
|
||||
])
|
||||
def test_services_publishing_a_host_port_are_flagged(text, port):
|
||||
[(kind, match)] = hy.scan_line(WF, text)
|
||||
assert kind == "host port" and port in match
|
||||
|
||||
|
||||
def test_host_port_rule_is_for_workflows_only():
|
||||
assert hy.scan_line("docker-compose.yml", " - 5432:5432") == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, ok", [
|
||||
(".github/workflows/ci.yml", True), (".gitea/workflows/check.yaml", True),
|
||||
("Dockerfile", True), ("api/Dockerfile.prod", True), ("docker/web.Dockerfile", True),
|
||||
("scripts/setup.sh", False), ("README.md", False), ("node_modules/x/Dockerfile", False),
|
||||
(".github/lint/x.yml", False),
|
||||
])
|
||||
def test_scope_is_ci_workflows_and_dockerfiles(path, ok):
|
||||
assert hy.path_ok(path) is ok
|
||||
|
||||
|
||||
def test_warn_mode_never_turns_red(monkeypatch):
|
||||
monkeypatch.setattr(hy, "MODE", "warn")
|
||||
state, desc, f = hy.status_for([hy.cg.Finding(WF, 12, "floating install", "npm install (use npm ci)")], True)
|
||||
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 CI hygiene issue in CI/Dockerfiles")
|
||||
|
||||
|
||||
def test_allow_file_loads_and_is_empty_today():
|
||||
assert hy.cg.load_allow(hy.ALLOW_FILE) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, text, want", [
|
||||
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv", "ghcr.io/astral-sh/uv:latest"), # Mail #147
|
||||
("Dockerfile", "FROM python:latest", "python:latest"),
|
||||
("Dockerfile", "FROM --platform=linux/amd64 node:latest AS web", "node:latest"),
|
||||
(WF, " image: postgres:latest", "postgres:latest"),
|
||||
(WF, " - uses: docker://ghcr.io/foo/bar:latest", "ghcr.io/foo/bar:latest"),
|
||||
])
|
||||
def test_latest_images_are_flagged(path, text, want):
|
||||
hits = hy.scan_line(path, text)
|
||||
assert ("floating image", want) in hits
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", [
|
||||
"COPY pyproject.toml uv.lock* ./", # Windy Mail #147
|
||||
"COPY package.json package-lock.json* ./",
|
||||
])
|
||||
def test_optional_lock_globs_are_flagged(text):
|
||||
assert [k for k, _ in hy.scan_line("Dockerfile", text)] == ["optional lock"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path, text", [
|
||||
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv"),
|
||||
("Dockerfile", "FROM python:3.12-slim"),
|
||||
("Dockerfile", "COPY pyproject.toml uv.lock ./"),
|
||||
("Dockerfile", "COPY src/*.py ./src/"),
|
||||
("Dockerfile", "RUN echo latest release notes"),
|
||||
])
|
||||
def test_pinned_images_and_real_locks_pass(path, text):
|
||||
assert hy.scan_line(path, text) == []
|
||||
@@ -182,3 +182,12 @@ def test_code_with_a_trailing_comment_still_counts():
|
||||
def test_windy_pro_desktop_is_byok_but_the_account_server_is_not():
|
||||
assert cg.allowed("windy-pro", "src/client/desktop/main.js", ALLOW)
|
||||
assert not cg.allowed("windy-pro", "account-server/src/routes/translations.ts", ALLOW)
|
||||
|
||||
|
||||
def test_a_commit_not_fetched_yet_is_skipped_not_an_error(tmp_path, monkeypatch):
|
||||
bare, sha = _repo(tmp_path, {"app/llm.py": "import anthropic\n"})
|
||||
monkeypatch.setattr(cg, "WORK", tmp_path)
|
||||
monkeypatch.setattr(cg, "CACHE", tmp_path / "cache.json")
|
||||
(tmp_path / "windy-chat.git").symlink_to(bare)
|
||||
assert cg.check("windy-chat", "f" * 40, "main", True) is None # pushed after the fetch
|
||||
assert [f.kind for f in cg.check("windy-chat", sha, "main", True)] == ["provider SDK"]
|
||||
|
||||
61
api/tests/test_guards_report.py
Normal file
61
api/tests/test_guards_report.py
Normal file
@@ -0,0 +1,61 @@
|
||||
"""guards_report: job attribution and the Grant-owned split."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
_spec = importlib.util.spec_from_file_location("guards_report", ROOT / "scripts" / "guards_report.py")
|
||||
gr = importlib.util.module_from_spec(_spec)
|
||||
sys.modules["guards_report"] = gr
|
||||
_spec.loader.exec_module(gr)
|
||||
|
||||
OWNED = yaml.safe_load((ROOT / "ci" / "grant-owned.yml").read_text())["grant_owned"]
|
||||
WF = """name: CI
|
||||
on:
|
||||
push:
|
||||
jobs:
|
||||
reality-check:
|
||||
runs-on: x
|
||||
steps:
|
||||
- run: npm install jsdom
|
||||
test-backend:
|
||||
runs-on: x
|
||||
steps:
|
||||
- run: pip install pytest
|
||||
"""
|
||||
|
||||
|
||||
def test_job_of_attributes_lines_to_their_job():
|
||||
assert gr.job_of(WF, 3) is None # `on:` block, not a job
|
||||
assert gr.job_of(WF, 8) == "reality-check"
|
||||
assert gr.job_of(WF, 12) == "test-backend"
|
||||
|
||||
|
||||
def test_windy_pro_desktop_jobs_and_paths_are_grant_owned():
|
||||
ci = ".github/workflows/ci.yml"
|
||||
assert gr.grant_owned("windy-pro", ci, "reality-check", OWNED)
|
||||
assert gr.grant_owned("windy-pro", ci, "build-electron", OWNED)
|
||||
assert not gr.grant_owned("windy-pro", ci, "test-backend", OWNED) # server side: 8c
|
||||
assert gr.grant_owned("windy-pro", ".github/workflows/release-mac.yml", None, OWNED)
|
||||
assert gr.grant_owned("windy-pro", "src/client/desktop/main.js", None, OWNED)
|
||||
assert not gr.grant_owned("windy-pro", "services/account-server/Dockerfile", None, OWNED)
|
||||
assert not gr.grant_owned("windy-chat", "src/client/desktop/main.js", None, OWNED)
|
||||
|
||||
|
||||
def test_render_splits_lane_and_grant_counts():
|
||||
F = gr.cg.Finding
|
||||
res = {"windy-pro": {"sha": "a" * 40, "compute": [],
|
||||
"hygiene": [(F("ci.yml", 8, "floating install", "npm install"), "reality-check", True),
|
||||
(F("ci.yml", 12, "floating install", "pip x"), "test-backend", False)]},
|
||||
"windy-git": {"sha": "b" * 40, "compute": [], "hygiene": []}}
|
||||
md = gr.render(res)
|
||||
assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md
|
||||
assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md
|
||||
assert "| windy-git | bbbbbbb | 0 | 0 | clean ✅ |" in md
|
||||
assert "(job reality-check)" in md
|
||||
@@ -382,3 +382,11 @@ def test_guard_that_cannot_run_posts_nothing(fake, monkeypatch):
|
||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard(None))
|
||||
bridge.post_compute_guard("windy-chat", SHA, "main", True)
|
||||
assert f.posted == []
|
||||
|
||||
|
||||
def test_ci_hygiene_posts_under_its_own_context(fake, monkeypatch):
|
||||
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "success", "description": "WARN 1"}])
|
||||
monkeypatch.setitem(sys.modules, "ci_hygiene", _Guard([_F()]))
|
||||
bridge.post_ci_hygiene("windy-chat", SHA, "main", True)
|
||||
# the compute-guard status with the same description must not suppress it
|
||||
assert [(p["context"], p["description"]) for p in f.posted] == [("windy-git/ci-hygiene", "WARN 1")]
|
||||
|
||||
5
ci/ci-hygiene-allow.yml
Normal file
5
ci/ci-hygiene-allow.yml
Normal file
@@ -0,0 +1,5 @@
|
||||
# CI hygiene allow-list: installs that may float, or services that may publish
|
||||
# a host port. House rule 6 (09-23): installs come from a lockfile. Every entry
|
||||
# is an exception and MUST say why. Paths are fnmatch globs from the repo root.
|
||||
# Owner: Windy Git lane (13); changes go through the orchestrator.
|
||||
allow: []
|
||||
15
ci/grant-owned.yml
Normal file
15
ci/grant-owned.yml
Normal file
@@ -0,0 +1,15 @@
|
||||
# Code Grant owns directly (orchestrator, 09-23): guard findings here are listed
|
||||
# SEPARATELY in the guards status page and never hold up "block". Changes to
|
||||
# these files are proposals for Grant / Windy Word 44, not a lane's fix.
|
||||
grant_owned:
|
||||
- repo: windy-pro
|
||||
reason: "Windy Word desktop (Electron) + its release/installer builds: Grant's, built from the Mac mini."
|
||||
paths:
|
||||
- "src/client/desktop/*"
|
||||
- "installer-v2/*"
|
||||
- ".github/workflows/build-windows.yml"
|
||||
- ".github/workflows/release-mac.yml"
|
||||
- ".github/workflows/build-installer.yml"
|
||||
- ".github/workflows/build-offline-installers.yml"
|
||||
jobs:
|
||||
".github/workflows/ci.yml": [reality-check, build-desktop, test-installer, build-electron]
|
||||
196
scripts/ci_hygiene.py
Normal file
196
scripts/ci_hygiene.py
Normal file
@@ -0,0 +1,196 @@
|
||||
#!/usr/bin/env python3
|
||||
"""CI hygiene guard: installs come from a lockfile, never "latest" (house rule 6).
|
||||
|
||||
A floating install lets CI test different versions than prod ships, and a
|
||||
rebuild silently changes prod. Windy Cloud's OpenAPI test failed on exactly
|
||||
that (fastapi 0.141.1 in CI vs 0.136.0 on the dev box) and all three Cloud
|
||||
cells floated in prod. Also flags services that publish a HOST port: every
|
||||
CI job shares one dind daemon, so two jobs publishing 5432 collide ("port is
|
||||
already allocated", Windy Mind runs 147/176).
|
||||
|
||||
WARN-ONLY (`windy-git/ci-hygiene`, green + "⚠ WARN"); CI_HYGIENE_MODE=block
|
||||
turns it red once the lanes report clean. Scans CI workflow files and
|
||||
Dockerfiles only. PR heads: lines the PR adds. Default branch: every line.
|
||||
|
||||
OK (not flagged):
|
||||
pip / uv pip install -r FILE (with or without --require-hashes), --no-deps,
|
||||
exact pins (tool==1.2.3), pip/setuptools/wheel upgrades
|
||||
uv sync --locked | --frozen npm ci
|
||||
npm install pkg@1.2.3 (every package exact-pinned)
|
||||
yarn install --frozen-lockfile / --immutable pnpm install --frozen-lockfile
|
||||
Also flagged: `:latest` images (FROM / COPY --from / image: / docker://) and
|
||||
`COPY uv.lock* ...`-style globs that build without the lock (Windy Mail #147).
|
||||
Exceptions: ci/ci-hygiene-allow.yml, one reason per entry.
|
||||
|
||||
python3 scripts/ci_hygiene.py report [repo ...]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import compute_guard as cg # noqa: E402 (shared walker, cache and allow-list loader)
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
ALLOW_FILE = Path(os.environ.get("CI_HYGIENE_ALLOW", ROOT / "ci" / "ci-hygiene-allow.yml"))
|
||||
MODE = os.environ.get("CI_HYGIENE_MODE", "warn")
|
||||
|
||||
# CI workflow files and Dockerfiles; never vendored copies.
|
||||
INCLUDE = re.compile(r"(^|/)\.(github|gitea)/workflows/[^/]+\.ya?ml$|(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
||||
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
||||
PREFILTER = (r"pip3? install|pip install|uv sync|npm (install|i )|yarn install|pnpm install"
|
||||
r"|^\s*-\s*['\"]?[0-9]+:[0-9]+|:latest|lock[^ ]*\*")
|
||||
|
||||
TOOLING = {"pip", "setuptools", "wheel"}
|
||||
DOCKER_FILE = re.compile(r"(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
||||
LATEST = re.compile(r"(?:^\s*FROM\s+(?:--platform=\S+\s+)?|--from=|image:\s*['\"]?|docker://)([\w./-]+):latest\b", re.I)
|
||||
LOCKNAME = re.compile(r"(uv\.lock|poetry\.lock|package-lock\.json|pnpm-lock\.yaml|yarn\.lock|requirements[^ ]*\.(txt|lock))", re.I)
|
||||
EXACT_PY = re.compile(r"^[A-Za-z0-9._-]+(\[[^\]]*\])?==[A-Za-z0-9.+!-]+$")
|
||||
EXACT_NPM = re.compile(r"^(@[^/@]+/)?[^/@]+@\d+\.\d+\.\d+([-+][0-9A-Za-z.-]+)?$")
|
||||
HOST_PORT = re.compile(r"^\s*-\s*['\"]?(\d{2,5}):(\d{2,5})['\"]?\s*(#.*)?$")
|
||||
PIP_VALUE_FLAGS = {"-c", "--constraint", "-i", "--index-url", "--extra-index-url", "-f",
|
||||
"--find-links", "--target", "-t", "--python", "--prefix", "--root", "--platform",
|
||||
"--python-version", "--implementation", "--abi", "--only-binary", "--no-binary"}
|
||||
|
||||
|
||||
def path_ok(path: str) -> bool:
|
||||
return bool(INCLUDE.search(path)) and not NEVER.search(path)
|
||||
|
||||
|
||||
def _commands(text: str) -> list[list[str]]:
|
||||
"""Split a shell line into simple commands (&&, ||, ;, |), tokenized."""
|
||||
out = []
|
||||
for part in re.split(r"&&|\|\||;|\|", text):
|
||||
try:
|
||||
toks = shlex.split(part, comments=True)
|
||||
except ValueError:
|
||||
toks = part.split()
|
||||
# Dockerfile RUN prefix / sudo / env-prefixed assignments
|
||||
while toks and (toks[0] in ("RUN", "sudo", "exec", "-", "run:", "command:")
|
||||
or re.match(r"^[A-Z_][A-Z0-9_]*=", toks[0])):
|
||||
toks = toks[1:]
|
||||
if toks:
|
||||
out.append(toks)
|
||||
return out
|
||||
|
||||
|
||||
def _pip_problem(args: list[str]) -> str | None:
|
||||
if "-r" in args or "--requirement" in args or any(a.startswith("--requirement=") for a in args):
|
||||
return None
|
||||
if "--no-deps" in args:
|
||||
return None
|
||||
pkgs, skip = [], False
|
||||
for a in args:
|
||||
if skip:
|
||||
skip = False
|
||||
continue
|
||||
if a in PIP_VALUE_FLAGS:
|
||||
skip = True
|
||||
continue
|
||||
if a.startswith("-") and a not in ("-e", "--editable"):
|
||||
continue
|
||||
if a in ("-e", "--editable"):
|
||||
continue
|
||||
pkgs.append(a)
|
||||
loose = [p for p in pkgs if not EXACT_PY.match(p) and p.split("[")[0].lower() not in TOOLING]
|
||||
if loose:
|
||||
return f"floating pip install: {' '.join(loose)[:40]}"
|
||||
return None
|
||||
|
||||
|
||||
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
||||
if cg.COMMENT.match(text):
|
||||
return []
|
||||
hits = []
|
||||
if "/workflows/" in path and HOST_PORT.match(text):
|
||||
hits.append(("host port", f"service publishes host port {HOST_PORT.match(text).group(1)} (shared dind)"))
|
||||
return hits
|
||||
# Windy Mail #147: a `:latest` build/tool image floats exactly like an
|
||||
# unpinned package, and `COPY uv.lock* ./` builds WITHOUT the lock when it
|
||||
# is missing instead of failing.
|
||||
m = LATEST.search(text)
|
||||
if m:
|
||||
hits.append(("floating image", f"{m.group(1)}:latest"))
|
||||
if DOCKER_FILE.search(path) and re.match(r"^\s*COPY\b", text, re.I):
|
||||
globbed = [t for t in text.split() if "*" in t and LOCKNAME.search(t)]
|
||||
if globbed:
|
||||
hits.append(("optional lock", f"COPY {globbed[0]} (must fail if the lock is missing)"))
|
||||
for toks in _commands(text):
|
||||
low = [t.lower() for t in toks]
|
||||
# pip install / python -m pip install / uv pip install
|
||||
for i in range(len(low) - 1):
|
||||
if os.path.basename(low[i]) in ("pip", "pip3") and low[i + 1] == "install":
|
||||
prob = _pip_problem(toks[i + 2:])
|
||||
if prob:
|
||||
hits.append(("floating install", prob))
|
||||
break
|
||||
if low[:2] == ["uv", "sync"] and not ({"--locked", "--frozen"} & set(low)):
|
||||
hits.append(("floating install", "uv sync without --locked/--frozen"))
|
||||
if low[:1] == ["npm"] and len(low) > 1 and low[1] in ("install", "i", "add"):
|
||||
pkgs = [t for t in toks[2:] if not t.startswith("-")]
|
||||
if not pkgs or not all(EXACT_NPM.match(p) for p in pkgs):
|
||||
hits.append(("floating install", f"npm {low[1]} {' '.join(pkgs)[:30]}".strip() + " (use npm ci)"))
|
||||
if low[:2] == ["yarn", "install"] and not ({"--frozen-lockfile", "--immutable"} & set(low)):
|
||||
hits.append(("floating install", "yarn install without --frozen-lockfile"))
|
||||
if low[:2] == ["pnpm", "install"] and "--frozen-lockfile" not in low:
|
||||
hits.append(("floating install", "pnpm install without --frozen-lockfile"))
|
||||
return hits
|
||||
|
||||
|
||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
||||
bare = cg.WORK / f"{repo}.git"
|
||||
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
||||
return None
|
||||
allow = cg.load_allow(ALLOW_FILE)
|
||||
rules = hashlib.sha256((PREFILTER + INCLUDE.pattern + EXACT_PY.pattern + EXACT_NPM.pattern).encode()).hexdigest()[:8]
|
||||
fp = cg._fingerprint(allow) + ":" + rules # hashlib, not hash(): hash() is per-process random
|
||||
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
||||
if is_default_head:
|
||||
return cg.cached_scan(f"hyg-tree:{repo}:{sha}:{fp}",
|
||||
lambda: cg.scan_tree(repo, bare, sha, allow, prefilter=PREFILTER, **kw))
|
||||
return cg.cached_scan(f"hyg-pr:{repo}:{sha}:{fp}",
|
||||
lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow, **kw))
|
||||
|
||||
|
||||
def status_for(findings, whole_tree: bool):
|
||||
scope = "in CI/Dockerfiles" if whole_tree else "added"
|
||||
if not findings:
|
||||
return "success", f"OK: no floating install or host-port service {scope}", None
|
||||
f = findings[0]
|
||||
n = len(findings)
|
||||
state = "failure" if MODE == "block" else "success"
|
||||
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
||||
return state, f"{lead}: {n} CI hygiene issue{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
||||
|
||||
|
||||
def report(repos: list[str]) -> int:
|
||||
allow = cg.load_allow(ALLOW_FILE)
|
||||
total = 0
|
||||
for repo in repos:
|
||||
bare = cg.WORK / f"{repo}.git"
|
||||
if not bare.is_dir():
|
||||
print(f"## {repo}: no sync clone, skipped")
|
||||
continue
|
||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||
sha = cg._git(bare, "rev-parse", head).strip()
|
||||
fs = cg.scan_tree(repo, bare, sha, allow, line_fn=scan_line, path_ok=path_ok, prefilter=PREFILTER)
|
||||
total += len(fs)
|
||||
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} issue(s)")
|
||||
for f in fs:
|
||||
print(f" {f.path}:{f.line} [{f.kind}] {f.match}")
|
||||
print(f"TOTAL {total}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
||||
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
||||
p.name.removesuffix(".git") for p in cg.WORK.glob("*.git"))
|
||||
sys.exit(report(sys.argv[2:] or default))
|
||||
sys.exit(__doc__)
|
||||
@@ -130,11 +130,20 @@ def _git(bare: Path, *args: str) -> str:
|
||||
).stdout
|
||||
|
||||
|
||||
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict]) -> list[Finding]:
|
||||
def _default_path_ok(path: str) -> bool:
|
||||
return not SKIP.search(path)
|
||||
|
||||
|
||||
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict], *, line_fn=None,
|
||||
path_ok=None, prefilter: str | None = None) -> list[Finding]:
|
||||
"""Every line in the tree at `sha` (default branch: the baseline)."""
|
||||
# A cheap prefilter by git, then the real rules in Python.
|
||||
pre = "|".join([re.escape(h) for h in HOSTS] + KEYS + ["anthropic", "openai", "groq", "mistral",
|
||||
"generativeai", "genai", "cohere", "together", "cerebras", "litellm"])
|
||||
# Other guards (ci_hygiene) reuse this walker with their own line rules.
|
||||
line_fn = line_fn or scan_line
|
||||
path_ok = path_ok or _default_path_ok
|
||||
pre = prefilter or "|".join([re.escape(h) for h in HOSTS] + KEYS + [
|
||||
"anthropic", "openai", "groq", "mistral", "generativeai", "genai", "cohere",
|
||||
"together", "cerebras", "litellm"])
|
||||
try:
|
||||
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
||||
except subprocess.CalledProcessError as e:
|
||||
@@ -148,24 +157,26 @@ def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict]) -> list[Findin
|
||||
_, path, line, text = raw.split(":", 3)
|
||||
except ValueError:
|
||||
continue
|
||||
if SKIP.search(path) or allowed(repo, path, allow):
|
||||
if not path_ok(path) or allowed(repo, path, allow):
|
||||
continue
|
||||
for kind, match in scan_line(path, text):
|
||||
for kind, match in line_fn(path, text):
|
||||
found.append(Finding(path, int(line), kind, match))
|
||||
return found
|
||||
|
||||
|
||||
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict]) -> list[Finding]:
|
||||
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict], **kw) -> list[Finding]:
|
||||
"""Only the lines a PR adds, vs its merge-base with the default branch."""
|
||||
mb = _git(bare, "merge-base", base_ref, sha).strip()
|
||||
diff = _git(bare, "diff", "-U0", "--no-color", "--no-ext-diff", mb, sha)
|
||||
return parse_added(repo, diff, allow)
|
||||
return parse_added(repo, diff, allow, **kw)
|
||||
|
||||
|
||||
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||
|
||||
|
||||
def parse_added(repo: str, diff: str, allow: list[dict]) -> list[Finding]:
|
||||
def parse_added(repo: str, diff: str, allow: list[dict], *, line_fn=None, path_ok=None) -> list[Finding]:
|
||||
line_fn = line_fn or scan_line
|
||||
path_ok = path_ok or _default_path_ok
|
||||
found, path, line = [], None, 0
|
||||
for raw in diff.splitlines():
|
||||
if raw.startswith("+++ "):
|
||||
@@ -179,8 +190,8 @@ def parse_added(repo: str, diff: str, allow: list[dict]) -> list[Finding]:
|
||||
if path is None or raw.startswith("--- "):
|
||||
continue
|
||||
if raw.startswith("+"):
|
||||
if not (SKIP.search(path) or allowed(repo, path, allow)):
|
||||
for kind, match in scan_line(path, raw[1:]):
|
||||
if path_ok(path) and not allowed(repo, path, allow):
|
||||
for kind, match in line_fn(path, raw[1:]):
|
||||
found.append(Finding(path, line, kind, match))
|
||||
line += 1
|
||||
return found
|
||||
@@ -214,10 +225,21 @@ def cached_scan(key: str, fn) -> list[Finding]:
|
||||
return result
|
||||
|
||||
|
||||
def fetched(bare: Path, sha: str) -> bool:
|
||||
"""Is `sha` in the sync clone yet? The bridge learns PR / default heads from
|
||||
GitHub's API AFTER the sync fetched, so a push in between is simply not here
|
||||
until the next 5-min cycle. That is a race, not an error: skip quietly."""
|
||||
try:
|
||||
_git(bare, "cat-file", "-e", f"{sha}^{{commit}}")
|
||||
return True
|
||||
except subprocess.CalledProcessError:
|
||||
return False
|
||||
|
||||
|
||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> list[Finding] | None:
|
||||
"""Findings for one commit, or None when the guard can't run (never a fake OK)."""
|
||||
bare = WORK / f"{repo}.git"
|
||||
if not bare.is_dir():
|
||||
if not bare.is_dir() or not fetched(bare, sha):
|
||||
return None
|
||||
allow = load_allow()
|
||||
fp = _fingerprint(allow)
|
||||
|
||||
124
scripts/guards_report.py
Normal file
124
scripts/guards_report.py
Normal file
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Live status of the repo guards (compute-guard + ci-hygiene) as one markdown page.
|
||||
|
||||
Scans every bridged repo's DEFAULT branch with both guards and renders what is
|
||||
left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml:
|
||||
windy-pro's desktop app and its build jobs) are listed in their OWN section and
|
||||
do not count against "ready to block": those are proposals for Grant, not a
|
||||
lane's fix (orchestrator, 09-23).
|
||||
|
||||
sudo python3 scripts/guards_report.py > GUARDS_STATUS.md
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import fnmatch
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import ci_hygiene as hy # noqa: E402
|
||||
import compute_guard as cg # noqa: E402
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml"))
|
||||
REPOS = os.environ.get("BRIDGE_REPOS", "").split() or [
|
||||
"windy-chat", "windy-mail", "windy-calendar", "Windy-Clone", "WindyCloud", "windy-search",
|
||||
"windy-connect", "windy-drops", "windy-code-web", "windy-code", "windy-traveler",
|
||||
"windy-registry", "eternitas", "windy-translate", "windytranslate-site", "windytraveler-site",
|
||||
"windy-hand", "windy-cloud-sites", "windy-cloud-domains", "windy-cloud-vps", "windytalk",
|
||||
"windy-pro", "windy-mind", "windy-git"]
|
||||
JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
|
||||
|
||||
|
||||
def job_of(text: str, line: int) -> str | None:
|
||||
"""The workflow job a line belongs to (2-space keys under `jobs:`)."""
|
||||
in_jobs, job = False, None
|
||||
for i, raw in enumerate(text.splitlines(), 1):
|
||||
if raw.startswith("jobs:"):
|
||||
in_jobs = True
|
||||
elif in_jobs and JOB.match(raw):
|
||||
job = JOB.match(raw).group(1)
|
||||
elif raw and not raw[0].isspace() and not raw.startswith("jobs:"):
|
||||
in_jobs = False
|
||||
if i == line:
|
||||
return job if in_jobs else None
|
||||
return None
|
||||
|
||||
|
||||
def grant_owned(repo: str, path: str, job: str | None, owned: list[dict]) -> bool:
|
||||
for e in owned:
|
||||
if e["repo"] != repo:
|
||||
continue
|
||||
if any(fnmatch.fnmatch(path, g) for g in e.get("paths") or []):
|
||||
return True
|
||||
if job and job in (e.get("jobs") or {}).get(path, []):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def scan(repo: str, owned: list[dict]):
|
||||
bare = cg.WORK / f"{repo}.git"
|
||||
if not bare.is_dir():
|
||||
return None
|
||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
||||
sha = cg._git(bare, "rev-parse", head).strip()
|
||||
out = {"sha": sha, "compute": [], "hygiene": []}
|
||||
texts: dict[str, str] = {}
|
||||
for key, fs in (("compute", cg.check(repo, sha, head, True) or []),
|
||||
("hygiene", hy.check(repo, sha, head, True) or [])):
|
||||
for f in fs:
|
||||
job = None
|
||||
if "/workflows/" in f.path:
|
||||
if f.path not in texts:
|
||||
texts[f.path] = cg._git(bare, "show", f"{sha}:{f.path}")
|
||||
job = job_of(texts[f.path], f.line)
|
||||
out[key].append((f, job, grant_owned(repo, f.path, job, owned)))
|
||||
return out
|
||||
|
||||
|
||||
def render(results: dict) -> str:
|
||||
now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime())
|
||||
lane = {k: 0 for k in ("compute", "hygiene")}
|
||||
grant = {k: 0 for k in ("compute", "hygiene")}
|
||||
for r in results.values():
|
||||
for k in lane:
|
||||
lane[k] += sum(1 for _, _, g in r[k] if not g)
|
||||
grant[k] += sum(1 for _, _, g in r[k] if g)
|
||||
L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)",
|
||||
"_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. "
|
||||
"Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "",
|
||||
"| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|",
|
||||
f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |",
|
||||
f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |",
|
||||
"", "## By repo (lane-owned)", "| Repo | head | compute | hygiene | first items |", "|---|---|---|---|---|"]
|
||||
for repo, r in sorted(results.items()):
|
||||
c = [x for x in r["compute"] if not x[2]]
|
||||
h = [x for x in r["hygiene"] if not x[2]]
|
||||
items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (c + h)[:3]) or "clean ✅"
|
||||
L.append(f"| {repo} | {r['sha'][:7]} | {len(c)} | {len(h)} | {items} |")
|
||||
L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"]
|
||||
g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene")
|
||||
for f, job, own in r[k] if own]
|
||||
L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"]
|
||||
return "\n".join(L) + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
owned = (yaml.safe_load(OWNED.read_text()) or {}).get("grant_owned") or []
|
||||
results = {}
|
||||
for repo in REPOS:
|
||||
r = scan(repo, owned)
|
||||
if r is not None:
|
||||
results[repo] = r
|
||||
sys.stdout.write(render(results))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -344,34 +344,45 @@ def post_statuses(repo: str, sha: str) -> None:
|
||||
|
||||
|
||||
GUARD_CTX = "windy-git/compute-guard"
|
||||
HYGIENE_CTX = "windy-git/ci-hygiene"
|
||||
|
||||
|
||||
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only).
|
||||
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only)."""
|
||||
_post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head)
|
||||
|
||||
Non-fatal and never a fake OK: if the guard can't run, nothing is posted.
|
||||
"""
|
||||
|
||||
def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||
"""House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only)."""
|
||||
_post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head)
|
||||
|
||||
|
||||
def _post_guard(modname: str, ctx: str, repo: str, sha: str, default_branch: str,
|
||||
is_default_head: bool) -> None:
|
||||
"""One code path for every repo-scanning guard. Non-fatal and never a fake OK:
|
||||
if the guard can't run, nothing is posted."""
|
||||
try:
|
||||
import compute_guard as cg # same directory; loaded lazily so the bridge never depends on it
|
||||
import importlib
|
||||
|
||||
findings = cg.check(repo, sha, default_branch, is_default_head)
|
||||
except Exception as e: # noqa: BLE001 — the guard must never break CI signals
|
||||
print(f" {repo}@{sha[:7]} compute-guard skipped ({type(e).__name__}: {str(e)[:80]})")
|
||||
g = importlib.import_module(modname) # same directory; lazy so the bridge never depends on it
|
||||
findings = g.check(repo, sha, default_branch, is_default_head)
|
||||
except Exception as e: # noqa: BLE001 — a guard must never break CI signals
|
||||
print(f" {repo}@{sha[:7]} {ctx} skipped ({type(e).__name__}: {str(e)[:80]})")
|
||||
return
|
||||
if findings is None:
|
||||
return
|
||||
state, desc, first = cg.status_for(findings, whole_tree=is_default_head)
|
||||
state, desc, first = g.status_for(findings, whole_tree=is_default_head)
|
||||
st, existing = github("GET", f"/repos/{GH_OWNER}/{repo}/commits/{sha}/statuses?per_page=100")
|
||||
for s in existing or []: # newest first: compare the latest guard status only
|
||||
if s["context"] == GUARD_CTX:
|
||||
if s["context"] == ctx:
|
||||
if (s["state"], s.get("description")) == (state, desc):
|
||||
return
|
||||
break
|
||||
url = (f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}/{first.path}#L{first.line}"
|
||||
if first else f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}")
|
||||
st, _ = github("POST", f"/repos/{GH_OWNER}/{repo}/statuses/{sha}",
|
||||
{"state": state, "context": GUARD_CTX, "description": desc, "target_url": url})
|
||||
print(f" {repo}@{sha[:7]} {GUARD_CTX} = {state} ({len(findings)} finding(s)) -> {st}")
|
||||
{"state": state, "context": ctx, "description": desc, "target_url": url})
|
||||
print(f" {repo}@{sha[:7]} {ctx} = {state} ({len(findings)} finding(s)) -> {st}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
@@ -392,6 +403,7 @@ def main() -> int:
|
||||
for sha in dict.fromkeys(shas):
|
||||
post_statuses(repo, sha)
|
||||
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
||||
post_ci_hygiene(repo, sha, default_branch, sha == default_head)
|
||||
except Exception as e: # one repo's failure must not hide the others'
|
||||
print(f" FAILED {repo}: {e}")
|
||||
failed = 1
|
||||
|
||||
Reference in New Issue
Block a user