Compare commits
1 Commits
1da4d39c0b
...
ci-inputs-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a2daca61ef |
@@ -51,10 +51,9 @@ jobs:
|
|||||||
- name: install
|
- name: install
|
||||||
run: |
|
run: |
|
||||||
python3 --version
|
python3 --version
|
||||||
# From uv.lock, never floating: CI tests exactly what the image ships.
|
python3 -m venv .venv
|
||||||
# --locked also FAILS if pyproject.toml changed without re-locking.
|
.venv/bin/pip install -q --upgrade pip
|
||||||
python3 -m pip install -q uv==0.12.5
|
.venv/bin/pip install -e ".[dev]"
|
||||||
uv sync --locked --extra dev
|
|
||||||
|
|
||||||
- name: lint
|
- name: lint
|
||||||
run: .venv/bin/ruff check api scripts
|
run: .venv/bin/ruff check api scripts
|
||||||
|
|||||||
15
Dockerfile
15
Dockerfile
@@ -10,19 +10,10 @@ WORKDIR /app
|
|||||||
RUN apt-get update && apt-get install -y --no-install-recommends git curl \
|
RUN apt-get update && apt-get install -y --no-install-recommends git curl \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# Dependencies come from uv.lock, hash-pinned, never "latest at build time".
|
COPY pyproject.toml ./
|
||||||
# Floating installs meant a rebuild could ship different fastapi/starlette/
|
RUN pip install --no-cache-dir -e .
|
||||||
# pydantic than CI tested (Windy Cloud's OpenAPI drift, 09-23). The lock was
|
|
||||||
# cut to exactly what prod ran then. uv only exports; pip installs, so the
|
|
||||||
# image layout (system python, uvicorn on PATH) is unchanged.
|
|
||||||
COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv
|
|
||||||
COPY pyproject.toml uv.lock ./
|
|
||||||
RUN uv export --frozen --no-dev --no-emit-project -o /tmp/requirements.txt \
|
|
||||||
&& pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \
|
|
||||||
&& rm /tmp/requirements.txt
|
|
||||||
COPY api ./api
|
|
||||||
RUN pip install --no-cache-dir --no-deps -e .
|
|
||||||
|
|
||||||
|
COPY api ./api
|
||||||
COPY alembic ./alembic
|
COPY alembic ./alembic
|
||||||
COPY alembic.ini ./
|
COPY alembic.ini ./
|
||||||
COPY scripts ./scripts
|
COPY scripts ./scripts
|
||||||
|
|||||||
@@ -1,164 +0,0 @@
|
|||||||
"""CI hygiene guard (house rule 6): lockfile-only installs, no host-port services."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
sys.path.insert(0, str(ROOT / "scripts"))
|
|
||||||
_spec = importlib.util.spec_from_file_location("ci_hygiene", ROOT / "scripts" / "ci_hygiene.py")
|
|
||||||
hy = importlib.util.module_from_spec(_spec)
|
|
||||||
sys.modules["ci_hygiene"] = hy
|
|
||||||
_spec.loader.exec_module(hy)
|
|
||||||
|
|
||||||
WF = ".github/workflows/ci.yml"
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, text", [
|
|
||||||
(WF, " .venv/bin/pip install -e \".[dev]\""), # windy-git's own, before e64a1b5
|
|
||||||
("Dockerfile", "RUN pip install --no-cache-dir -e ."), # windy-git image, before e64a1b5
|
|
||||||
(WF, " - run: uv pip install -e \".[dev]\""), # WindyCloud #109's CI
|
|
||||||
(WF, " run: pip install fastapi uvicorn"),
|
|
||||||
(WF, " - run: uv sync --all-extras"), # windy-mind style, not locked
|
|
||||||
(WF, " - run: npm install"), # windy-drops / windytalk
|
|
||||||
(WF, " - run: npm install --no-save --no-audit --no-fund jsdom"), # windy-pro reality-check
|
|
||||||
(WF, " - run: yarn install"),
|
|
||||||
(WF, " - run: cd web && pnpm install"),
|
|
||||||
("docker/api.Dockerfile", "RUN apt-get update && pip install requests"),
|
|
||||||
])
|
|
||||||
def test_floating_installs_are_flagged(path, text):
|
|
||||||
assert [k for k, _ in hy.scan_line(path, text)] == ["floating install"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, text", [
|
|
||||||
(WF, " python3 -m pip install -q uv==0.12.5"), # exact tool pin
|
|
||||||
(WF, " uv sync --locked --extra dev"),
|
|
||||||
(WF, " - run: uv sync --frozen"),
|
|
||||||
(WF, " - run: npm ci"),
|
|
||||||
(WF, " - run: npm install --no-save jsdom@24.1.0"),
|
|
||||||
(WF, " - run: pip install -r requirements.lock --require-hashes"),
|
|
||||||
(WF, " - run: pip install -r requirements.txt"),
|
|
||||||
("Dockerfile", " && pip install --no-cache-dir --require-hashes -r /tmp/requirements.txt \\\\"),
|
|
||||||
("Dockerfile", "RUN pip install --no-cache-dir --no-deps -e ."), # project only, deps from the lock
|
|
||||||
(WF, " .venv/bin/pip install -q --upgrade pip"),
|
|
||||||
(WF, " - run: yarn install --frozen-lockfile"),
|
|
||||||
(WF, " # - run: npm install (commented out)"),
|
|
||||||
(WF, " - run: echo 'pip is great'"),
|
|
||||||
])
|
|
||||||
def test_locked_or_pinned_installs_pass(path, text):
|
|
||||||
assert hy.scan_line(path, text) == []
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text, port", [
|
|
||||||
(" - 5432:5432", "5432"), # windy-mind / eternitas (collided 09-23)
|
|
||||||
(" - '15432:5432'", "15432"), # WindyCloud
|
|
||||||
(' - "6379:6379"', "6379"),
|
|
||||||
])
|
|
||||||
def test_services_publishing_a_host_port_are_flagged(text, port):
|
|
||||||
[(kind, match)] = hy.scan_line(WF, text)
|
|
||||||
assert kind == "host port" and port in match
|
|
||||||
|
|
||||||
|
|
||||||
def test_host_port_rule_is_for_workflows_only():
|
|
||||||
assert hy.scan_line("docker-compose.yml", " - 5432:5432") == []
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, ok", [
|
|
||||||
(".github/workflows/ci.yml", True), (".gitea/workflows/check.yaml", True),
|
|
||||||
("Dockerfile", True), ("api/Dockerfile.prod", True), ("docker/web.Dockerfile", True),
|
|
||||||
("scripts/setup.sh", False), ("README.md", False), ("node_modules/x/Dockerfile", False),
|
|
||||||
(".github/lint/x.yml", False),
|
|
||||||
])
|
|
||||||
def test_scope_is_ci_workflows_and_dockerfiles(path, ok):
|
|
||||||
assert hy.path_ok(path) is ok
|
|
||||||
|
|
||||||
|
|
||||||
def test_warn_mode_never_turns_red(monkeypatch):
|
|
||||||
monkeypatch.setattr(hy, "MODE", "warn")
|
|
||||||
state, desc, f = hy.status_for([hy.cg.Finding(WF, 12, "floating install", "npm install (use npm ci)")], True)
|
|
||||||
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 CI hygiene issue in CI/Dockerfiles")
|
|
||||||
|
|
||||||
|
|
||||||
def test_allow_file_is_line_scoped_exceptions_only():
|
|
||||||
"""Every exception is line-scoped (`matches`), so an allowed file can't hide a
|
|
||||||
NEW floating install or docker step. Today: windy-pro's if:false deploy job."""
|
|
||||||
allow = hy.cg.load_allow(hy.ALLOW_FILE)
|
|
||||||
assert [(e["repo"], e["paths"]) for e in allow] == [("windy-pro", [".github/workflows/ci.yml"])]
|
|
||||||
assert all(e.get("matches") for e in allow)
|
|
||||||
ok = " run: docker build -f account-server/Dockerfile -t windy-pro:${{ github.sha }} ."
|
|
||||||
new = " run: docker build -t windy-pro-api ."
|
|
||||||
assert hy.cg.allowed("windy-pro", WF, allow, ok)
|
|
||||||
assert not hy.cg.allowed("windy-pro", WF, allow, new)
|
|
||||||
assert not hy.cg.allowed("windy-chat", WF, allow, ok)
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, text, want", [
|
|
||||||
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv", "ghcr.io/astral-sh/uv:latest"), # Mail #147
|
|
||||||
("Dockerfile", "FROM python:latest", "python:latest"),
|
|
||||||
("Dockerfile", "FROM --platform=linux/amd64 node:latest AS web", "node:latest"),
|
|
||||||
(WF, " image: postgres:latest", "postgres:latest"),
|
|
||||||
(WF, " - uses: docker://ghcr.io/foo/bar:latest", "ghcr.io/foo/bar:latest"),
|
|
||||||
])
|
|
||||||
def test_latest_images_are_flagged(path, text, want):
|
|
||||||
hits = hy.scan_line(path, text)
|
|
||||||
assert ("floating image", want) in hits
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text", [
|
|
||||||
"COPY pyproject.toml uv.lock* ./", # Windy Mail #147
|
|
||||||
"COPY package.json package-lock.json* ./",
|
|
||||||
])
|
|
||||||
def test_optional_lock_globs_are_flagged(text):
|
|
||||||
assert [k for k, _ in hy.scan_line("Dockerfile", text)] == ["optional lock"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, text", [
|
|
||||||
("Dockerfile", "COPY --from=ghcr.io/astral-sh/uv:0.12.5 /uv /usr/local/bin/uv"),
|
|
||||||
("Dockerfile", "FROM python:3.12-slim"),
|
|
||||||
("Dockerfile", "COPY pyproject.toml uv.lock ./"),
|
|
||||||
("Dockerfile", "COPY src/*.py ./src/"),
|
|
||||||
("Dockerfile", "RUN echo latest release notes"),
|
|
||||||
])
|
|
||||||
def test_pinned_images_and_real_locks_pass(path, text):
|
|
||||||
assert hy.scan_line(path, text) == []
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text", [
|
|
||||||
" - run: docker compose -f docker-compose.yml -f docker-compose.ci.yml build", # eternitas ci/build
|
|
||||||
" run: docker build -t windy-mail .",
|
|
||||||
" - run: docker-compose up -d",
|
|
||||||
" run: docker buildx build --load .",
|
|
||||||
" - uses: docker/build-push-action@v6",
|
|
||||||
])
|
|
||||||
def test_docker_in_ci_is_flagged_with_the_fix(text):
|
|
||||||
hits = hy.scan_line(WF, text)
|
|
||||||
assert [k for k, _ in hits] == ["needs docker"]
|
|
||||||
assert "no-Docker smoke test" in hits[0][1]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("path, text", [
|
|
||||||
("Dockerfile", "RUN docker build ."), # not a workflow
|
|
||||||
(WF, " run: ssh host 'docker compose up -d'"), # remote host has a daemon
|
|
||||||
(WF, " # docker compose build"), # comment
|
|
||||||
(WF, " run: echo docker build"),
|
|
||||||
])
|
|
||||||
def test_docker_not_flagged_outside_ci_steps(path, text):
|
|
||||||
assert [k for k, _ in hy.scan_line(path, text) if k == "needs docker"] == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_needs_docker_skips_workflows_disabled_on_windy_git(monkeypatch):
|
|
||||||
"""deploy.yml runs on the target host (a real daemon); Gitea has it disabled here."""
|
|
||||||
F = hy.cg.Finding
|
|
||||||
monkeypatch.setattr(hy, "_DISABLED", {"eternitas": {"deploy.yml"}})
|
|
||||||
got = hy._runs_here("Eternitas", [
|
|
||||||
F(".github/workflows/deploy.yml", 70, "needs docker", "docker compose in CI"),
|
|
||||||
F(".github/workflows/ci.yml", 176, "needs docker", "docker compose in CI"),
|
|
||||||
F(".github/workflows/deploy.yml", 12, "floating install", "npm install"),
|
|
||||||
])
|
|
||||||
assert [(f.path.rsplit("/", 1)[1], f.kind) for f in got] == [
|
|
||||||
("ci.yml", "needs docker"), ("deploy.yml", "floating install")]
|
|
||||||
assert hy._runs_here("eternitas", None) is None
|
|
||||||
@@ -182,50 +182,3 @@ def test_code_with_a_trailing_comment_still_counts():
|
|||||||
def test_windy_pro_desktop_is_byok_but_the_account_server_is_not():
|
def test_windy_pro_desktop_is_byok_but_the_account_server_is_not():
|
||||||
assert cg.allowed("windy-pro", "src/client/desktop/main.js", ALLOW)
|
assert cg.allowed("windy-pro", "src/client/desktop/main.js", ALLOW)
|
||||||
assert not cg.allowed("windy-pro", "account-server/src/routes/translations.ts", ALLOW)
|
assert not cg.allowed("windy-pro", "account-server/src/routes/translations.ts", ALLOW)
|
||||||
|
|
||||||
|
|
||||||
def test_a_commit_not_fetched_yet_is_skipped_not_an_error(tmp_path, monkeypatch):
|
|
||||||
bare, sha = _repo(tmp_path, {"app/llm.py": "import anthropic\n"})
|
|
||||||
monkeypatch.setattr(cg, "WORK", tmp_path)
|
|
||||||
monkeypatch.setattr(cg, "CACHE", tmp_path / "cache.json")
|
|
||||||
(tmp_path / "windy-chat.git").symlink_to(bare)
|
|
||||||
assert cg.check("windy-chat", "f" * 40, "main", True) is None # pushed after the fetch
|
|
||||||
assert [f.kind for f in cg.check("windy-chat", sha, "main", True)] == ["provider SDK"]
|
|
||||||
|
|
||||||
|
|
||||||
KEYCHAIN = "src/client/web/src/pages/panels/MindKeychain.jsx"
|
|
||||||
|
|
||||||
|
|
||||||
def test_scoped_allow_admits_only_the_oauth_endpoints():
|
|
||||||
ok = [
|
|
||||||
" window.location.href = `https://openrouter.ai/auth?callback_url=${encodeURIComponent(callback)}`",
|
|
||||||
" const res = await fetch('https://openrouter.ai/api/v1/auth/keys', {",
|
|
||||||
]
|
|
||||||
for line in ok:
|
|
||||||
assert cg.allowed("windy-pro", KEYCHAIN, ALLOW, line)
|
|
||||||
# an inference call smuggled into the same file still flags
|
|
||||||
assert not cg.allowed("windy-pro", KEYCHAIN, ALLOW,
|
|
||||||
" await fetch('https://openrouter.ai/api/v1/chat/completions', {")
|
|
||||||
# a scoped entry never allows a line it can't see
|
|
||||||
assert not cg.allowed("windy-pro", KEYCHAIN, ALLOW)
|
|
||||||
|
|
||||||
|
|
||||||
def test_scoped_allow_in_a_real_diff():
|
|
||||||
diff = f"""--- /dev/null
|
|
||||||
+++ b/{KEYCHAIN}
|
|
||||||
@@ -0,0 +1,3 @@
|
|
||||||
+ window.location.href = `https://openrouter.ai/auth?callback_url=x`
|
|
||||||
+ const res = await fetch('https://openrouter.ai/api/v1/auth/keys', {{
|
|
||||||
+ await fetch('https://openrouter.ai/api/v1/chat/completions', {{
|
|
||||||
"""
|
|
||||||
fs = cg.parse_added("windy-pro", diff, ALLOW)
|
|
||||||
assert [(f.line, f.match) for f in fs] == [(3, "openrouter.ai")]
|
|
||||||
|
|
||||||
|
|
||||||
def test_block_mode_never_blocks_grant_owned(monkeypatch):
|
|
||||||
monkeypatch.setattr(cg, "MODE", "block")
|
|
||||||
g = cg.Finding("src/client/desktop/x.js", 9, "provider host", "api.openai.com")
|
|
||||||
state, desc, f = cg.status_for([], whole_tree=True, grant=[g])
|
|
||||||
assert state == "success" and desc.startswith("⚠ WARN (Grant-owned, not blocking): 1") and f is g
|
|
||||||
lane = cg.Finding("a.py", 3, "provider host", "x")
|
|
||||||
assert cg.status_for([lane], whole_tree=True, grant=[g])[0] == "failure"
|
|
||||||
|
|||||||
@@ -1,77 +0,0 @@
|
|||||||
"""guards_report: job attribution and the Grant-owned split."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
sys.path.insert(0, str(ROOT / "scripts"))
|
|
||||||
_spec = importlib.util.spec_from_file_location("guards_report", ROOT / "scripts" / "guards_report.py")
|
|
||||||
gr = importlib.util.module_from_spec(_spec)
|
|
||||||
sys.modules["guards_report"] = gr
|
|
||||||
_spec.loader.exec_module(gr)
|
|
||||||
|
|
||||||
OWNED = yaml.safe_load((ROOT / "ci" / "grant-owned.yml").read_text())["grant_owned"]
|
|
||||||
WF = """name: CI
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
jobs:
|
|
||||||
reality-check:
|
|
||||||
runs-on: x
|
|
||||||
steps:
|
|
||||||
- run: npm install jsdom
|
|
||||||
test-backend:
|
|
||||||
runs-on: x
|
|
||||||
steps:
|
|
||||||
- run: pip install pytest
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def test_job_of_attributes_lines_to_their_job():
|
|
||||||
assert gr.job_of(WF, 3) is None # `on:` block, not a job
|
|
||||||
assert gr.job_of(WF, 8) == "reality-check"
|
|
||||||
assert gr.job_of(WF, 12) == "test-backend"
|
|
||||||
|
|
||||||
|
|
||||||
def test_windy_pro_desktop_jobs_and_paths_are_grant_owned():
|
|
||||||
ci = ".github/workflows/ci.yml"
|
|
||||||
assert gr.grant_owned("windy-pro", ci, "reality-check", OWNED)
|
|
||||||
assert gr.grant_owned("windy-pro", ci, "build-electron", OWNED)
|
|
||||||
assert not gr.grant_owned("windy-pro", ci, "test-backend", OWNED) # server side: 8c
|
|
||||||
assert gr.grant_owned("windy-pro", ".github/workflows/release-mac.yml", None, OWNED)
|
|
||||||
assert gr.grant_owned("windy-pro", "src/client/desktop/main.js", None, OWNED)
|
|
||||||
assert not gr.grant_owned("windy-pro", "services/account-server/Dockerfile", None, OWNED)
|
|
||||||
assert not gr.grant_owned("windy-chat", "src/client/desktop/main.js", None, OWNED)
|
|
||||||
|
|
||||||
|
|
||||||
def test_render_splits_lane_and_grant_counts():
|
|
||||||
F = gr.cg.Finding
|
|
||||||
res = {"windy-pro": {"sha": "a" * 40, "compute": [],
|
|
||||||
"hygiene": [(F("ci.yml", 8, "floating install", "npm install"), "reality-check", True),
|
|
||||||
(F("ci.yml", 12, "floating install", "pip x"), "test-backend", False)]},
|
|
||||||
"windy-git": {"sha": "b" * 40, "compute": [], "hygiene": []}}
|
|
||||||
md = gr.render(res)
|
|
||||||
assert "| ci-hygiene (house rule 6) | 1 | 1 | ❌ not yet |" in md
|
|
||||||
assert "| compute-guard (Mind is the only door) | 0 | 0 | ✅ YES |" in md
|
|
||||||
assert "| windy-git | Windy Git | bbbbbbb | 0 | 0 | 0 | clean ✅ |" in md
|
|
||||||
assert "| windy-pro | Windy Hub | aaaaaaa |" in md # owner = session to message
|
|
||||||
assert "(job reality-check)" in md
|
|
||||||
|
|
||||||
|
|
||||||
def test_windy_pro_root_env_example_is_grant_owned_but_not_the_account_servers():
|
|
||||||
assert gr.grant_owned("windy-pro", ".env.example", None, OWNED)
|
|
||||||
assert not gr.grant_owned("windy-pro", "account-server/.env.example", None, OWNED)
|
|
||||||
|
|
||||||
|
|
||||||
def test_split_grant_sends_desktop_code_to_grant(monkeypatch):
|
|
||||||
F = gr.cg.Finding
|
|
||||||
fs = [F("src/client/desktop/main.js", 3, "provider host", "x"),
|
|
||||||
F("account-server/src/llm.ts", 5, "provider host", "y")]
|
|
||||||
lane, grant = gr.split_grant("windy-pro", "a" * 40, fs)
|
|
||||||
assert [f.path for f in grant] == ["src/client/desktop/main.js"]
|
|
||||||
assert [f.path for f in lane] == ["account-server/src/llm.ts"]
|
|
||||||
assert gr.split_grant("windy-chat", "a" * 40, fs) == (fs, [])
|
|
||||||
@@ -351,12 +351,10 @@ class _Guard:
|
|||||||
return self.findings
|
return self.findings
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def status_for(findings, whole_tree, grant=()):
|
def status_for(findings, whole_tree):
|
||||||
if not findings and grant:
|
|
||||||
return "success", f"GRANT-WARN {len(grant)}", grant[0]
|
|
||||||
if not findings:
|
if not findings:
|
||||||
return "success", "OK: clean", None
|
return "success", "OK: clean", None
|
||||||
return "failure", f"BLOCK {len(findings)}", findings[0]
|
return "success", f"WARN {len(findings)}", findings[0]
|
||||||
|
|
||||||
|
|
||||||
class _F:
|
class _F:
|
||||||
@@ -368,12 +366,12 @@ def test_guard_posts_warn_with_a_link_to_the_first_finding(fake, monkeypatch):
|
|||||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||||
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||||
assert [(p["context"], p["state"], p["description"]) for p in f.posted] == [
|
assert [(p["context"], p["state"], p["description"]) for p in f.posted] == [
|
||||||
("windy-git/compute-guard", "failure", "BLOCK 1")]
|
("windy-git/compute-guard", "success", "WARN 1")]
|
||||||
assert f.posted[0]["target_url"].endswith(f"/src/commit/{SHA}/app/llm.py#L7")
|
assert f.posted[0]["target_url"].endswith(f"/src/commit/{SHA}/app/llm.py#L7")
|
||||||
|
|
||||||
|
|
||||||
def test_guard_same_status_is_not_reposted(fake, monkeypatch):
|
def test_guard_same_status_is_not_reposted(fake, monkeypatch):
|
||||||
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "failure", "description": "BLOCK 1"}])
|
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "success", "description": "WARN 1"}])
|
||||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
||||||
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
bridge.post_compute_guard("windy-chat", SHA, "main", False)
|
||||||
assert f.posted == []
|
assert f.posted == []
|
||||||
@@ -384,80 +382,3 @@ def test_guard_that_cannot_run_posts_nothing(fake, monkeypatch):
|
|||||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard(None))
|
monkeypatch.setitem(sys.modules, "compute_guard", _Guard(None))
|
||||||
bridge.post_compute_guard("windy-chat", SHA, "main", True)
|
bridge.post_compute_guard("windy-chat", SHA, "main", True)
|
||||||
assert f.posted == []
|
assert f.posted == []
|
||||||
|
|
||||||
|
|
||||||
def test_ci_hygiene_posts_under_its_own_context(fake, monkeypatch):
|
|
||||||
f = fake(statuses=[{"context": "windy-git/compute-guard", "state": "failure", "description": "BLOCK 1"}])
|
|
||||||
monkeypatch.setitem(sys.modules, "ci_hygiene", _Guard([_F()]))
|
|
||||||
bridge.post_ci_hygiene("windy-chat", SHA, "main", True)
|
|
||||||
# the compute-guard status with the same description must not suppress it
|
|
||||||
assert [(p["context"], p["description"]) for p in f.posted] == [("windy-git/ci-hygiene", "BLOCK 1")]
|
|
||||||
|
|
||||||
|
|
||||||
def test_retargeted_pr_gets_a_fresh_mirror_on_the_new_base(fake):
|
|
||||||
# eternitas #167: stacked on fix/one-hallway, retargeted to main on GitHub.
|
|
||||||
gh = [{"number": 167, "title": "feat", "html_url": "u",
|
|
||||||
"head": {"ref": "feat/x", "sha": SHA, "repo": {"full_name": f"{bridge.GH_OWNER}/eternitas"}},
|
|
||||||
"base": {"ref": "main"}}]
|
|
||||||
wg = [{"number": 9, "title": "[GH#167] feat", "base": {"ref": "fix/one-hallway"}}]
|
|
||||||
f = fake(gh_prs=gh, wg_prs=wg)
|
|
||||||
assert bridge.sync_prs("eternitas") == [SHA]
|
|
||||||
assert f.closed == [f"/repos/{bridge.WG_OWNER}/eternitas/pulls/9"]
|
|
||||||
assert [(o["base"], o["head"]) for o in f.opened] == [("main", "feat/x")]
|
|
||||||
|
|
||||||
|
|
||||||
def test_unchanged_base_leaves_the_mirror_alone(fake):
|
|
||||||
gh = [{"number": 5, "title": "t", "html_url": "u",
|
|
||||||
"head": {"ref": "b", "sha": SHA, "repo": {"full_name": f"{bridge.GH_OWNER}/windy-chat"}},
|
|
||||||
"base": {"ref": "main"}}]
|
|
||||||
f = fake(gh_prs=gh, wg_prs=[{"number": 3, "title": "[GH#5] t", "base": {"ref": "main"}}])
|
|
||||||
bridge.sync_prs("windy-chat")
|
|
||||||
assert f.closed == [] and f.opened == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_named_no_daemon_job_is_not_posted_for_that_repo_only(fake, monkeypatch):
|
|
||||||
"""eternitas ci/build needs Docker but its name doesn't say so (option A, 09-23)."""
|
|
||||||
monkeypatch.setattr(bridge, "NO_DAEMON_NAMED", {"eternitas": {"ci/build"}})
|
|
||||||
runs = [_run(1, "ci.yml", "build", "failure"), _run(2, "ci.yml", "test", "success")]
|
|
||||||
f = fake(runs=runs)
|
|
||||||
bridge.post_statuses("eternitas", SHA)
|
|
||||||
assert [p["context"] for p in f.posted] == ["windy-git/ci/test"]
|
|
||||||
f2 = fake(runs=runs)
|
|
||||||
bridge.post_statuses("windy-chat", SHA)
|
|
||||||
assert sorted(p["context"] for p in f2.posted) == ["windy-git/ci/build", "windy-git/ci/test"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_default_no_daemon_named_is_empty():
|
|
||||||
"""eternitas converted its ci/build to a no-Docker ci/smoke (#179); nothing left."""
|
|
||||||
assert bridge.NO_DAEMON_NAMED == {}
|
|
||||||
|
|
||||||
|
|
||||||
def test_grant_owned_findings_never_block(fake, monkeypatch):
|
|
||||||
"""Orchestrator 09-23: compute-guard blocks lane-owned code only."""
|
|
||||||
f = fake()
|
|
||||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
|
||||||
monkeypatch.setitem(sys.modules, "guards_report",
|
|
||||||
type("GR", (), {"split_grant": staticmethod(lambda r, s, fs: ([], list(fs)))}))
|
|
||||||
bridge.post_compute_guard("windy-pro", SHA, "main", True)
|
|
||||||
assert [(p["state"], p["description"]) for p in f.posted] == [("success", "GRANT-WARN 1")]
|
|
||||||
|
|
||||||
|
|
||||||
def test_failed_grant_split_warns_instead_of_blocking(fake, monkeypatch):
|
|
||||||
def boom(*a):
|
|
||||||
raise RuntimeError("no bare clone")
|
|
||||||
|
|
||||||
f = fake()
|
|
||||||
monkeypatch.setitem(sys.modules, "compute_guard", _Guard([_F()]))
|
|
||||||
monkeypatch.setitem(sys.modules, "guards_report", type("GR", (), {"split_grant": staticmethod(boom)}))
|
|
||||||
bridge.post_compute_guard("windy-pro", SHA, "main", True)
|
|
||||||
assert [p["state"] for p in f.posted] == ["success"]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("name, hidden", [
|
|
||||||
("Docker Build", True), ("docker-build", True), ("Docker build", True), ("docker", True),
|
|
||||||
("Boot smoke (no Docker)", False), ("smoke (no-docker)", False), ("boot without Docker", False),
|
|
||||||
("smoke", False),
|
|
||||||
])
|
|
||||||
def test_no_docker_smoke_jobs_are_posted(name, hidden):
|
|
||||||
"""windy-search #96: its replacement job says "no Docker" and was hidden."""
|
|
||||||
assert bridge.needs_daemon("windy-search", "ci", name) is hidden
|
|
||||||
|
|||||||
@@ -1,148 +0,0 @@
|
|||||||
"""Secret guard: shapes, hash-only findings, allow by hash."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import importlib.util
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import pytest
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
|
||||||
sys.path.insert(0, str(ROOT / "scripts"))
|
|
||||||
_spec = importlib.util.spec_from_file_location("secret_guard", ROOT / "scripts" / "secret_guard.py")
|
|
||||||
sg = importlib.util.module_from_spec(_spec)
|
|
||||||
sys.modules["secret_guard"] = sg
|
|
||||||
_spec.loader.exec_module(sg)
|
|
||||||
ss = sg.ss
|
|
||||||
|
|
||||||
# Synthetic shapes only: none of these is a real credential.
|
|
||||||
TG = "1234567890:" + "A" * 35
|
|
||||||
CASES = [
|
|
||||||
("telegram bot token", f"TELEGRAM_BOT_TOKEN={TG}"),
|
|
||||||
("github token", "token = 'ghp_" + "a1" * 18 + "'"),
|
|
||||||
("aws access key", "aws_access_key_id = AKIA" + "ABCDEFGHIJKLMNOP"),
|
|
||||||
("slack token", "xoxb-" + "1234567890-abcdefghij"),
|
|
||||||
("anthropic key", "ANTHROPIC_API_KEY=sk-ant-" + "x" * 30),
|
|
||||||
("openai key", "OPENAI_API_KEY=sk-proj-" + "y" * 40),
|
|
||||||
("stripe live key", "STRIPE=sk_live_" + "z" * 24),
|
|
||||||
("google api key", "key=AIza" + "B" * 35),
|
|
||||||
("private key block", "-----BEGIN OPENSSH PRIVATE KEY-----"),
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("kind, text", CASES)
|
|
||||||
def test_each_shape_is_found_and_only_its_hash_is_kept(kind, text):
|
|
||||||
hits = sg.scan_line("app.py", text)
|
|
||||||
assert [k for k, _ in hits] == [kind]
|
|
||||||
match = hits[0][1]
|
|
||||||
assert match.startswith(f"{kind} #") and len(match.rsplit("#", 1)[1]) == 8
|
|
||||||
# house rule 10: the value itself must never appear in a finding
|
|
||||||
secret = text.split("=", 1)[-1].strip(" '")
|
|
||||||
assert secret not in match
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("text", [
|
|
||||||
"sha512-" + "Q" * 86 + "==", # lockfile integrity
|
|
||||||
"version: 12345678:abc", # short, not a token
|
|
||||||
"sk-ant-short", # too short
|
|
||||||
"re_test_register_sends_verification", # windy-pro's fake Resend key
|
|
||||||
"ANTHROPIC_API_KEY=", # a name, not a value
|
|
||||||
])
|
|
||||||
def test_non_secrets_are_not_flagged(text):
|
|
||||||
assert sg.scan_line("x", text) == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_anthropic_key_is_not_double_counted_as_openai():
|
|
||||||
assert [k for k, _ in sg.scan_line("x", "sk-ant-" + "q" * 40)] == ["anthropic key"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_allow_is_by_hash_only():
|
|
||||||
F = sg.cg.Finding
|
|
||||||
fake = F("tests/t.py", 3, "telegram bot token", f"telegram bot token #{ss.h8(TG)}")
|
|
||||||
real = F("tests/t.py", 9, "telegram bot token", "telegram bot token #deadbeef")
|
|
||||||
allow = {"windy-chat": {"hashes": {ss.h8(TG)}, "paths": []}}
|
|
||||||
assert sg._drop_allowed("windy-chat", [fake, real], allow) == [real]
|
|
||||||
assert sg._drop_allowed("windy-mail", [fake], allow) == [fake]
|
|
||||||
|
|
||||||
|
|
||||||
def test_private_key_blocks_are_allowed_by_path_never_by_hash():
|
|
||||||
F = sg.cg.Finding
|
|
||||||
hdr = "private key block #" + ss.h8("-----BEGIN PRIVATE KEY-----")
|
|
||||||
test_key = F("tests/keys/test.pem", 1, "private key block", hdr)
|
|
||||||
prod_key = F("deploy/prod.pem", 1, "private key block", hdr)
|
|
||||||
allow = {"r": {"hashes": {hdr.rsplit("#", 1)[1]}, "paths": [("tests/keys/*", {"private key block"})]}}
|
|
||||||
assert sg._drop_allowed("r", [test_key, prod_key], allow) == [prod_key]
|
|
||||||
|
|
||||||
|
|
||||||
def test_path_allow_cannot_cover_real_token_kinds(tmp_path):
|
|
||||||
bad = tmp_path / "a.yml"
|
|
||||||
bad.write_text("allow:\n - repo: r\n paths: [tests/*]\n kinds: [telegram bot token]\n reason: no\n")
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
sg.load_allow(bad)
|
|
||||||
|
|
||||||
|
|
||||||
def test_shipped_allow_file_never_excuses_the_real_leaked_tokens():
|
|
||||||
a = sg.load_allow()
|
|
||||||
every = set().union(*(v["hashes"] for v in a.values())) if a else set()
|
|
||||||
assert not {"1354fc9b", "d49dc2ba"} & every # real (now revoked) credentials: remove, never allow
|
|
||||||
|
|
||||||
|
|
||||||
def test_allow_file_loads_and_needs_reasons(tmp_path):
|
|
||||||
assert isinstance(sg.load_allow(), dict)
|
|
||||||
bad = tmp_path / "a.yml"
|
|
||||||
bad.write_text("allow:\n - repo: r\n hashes: [abcd1234]\n")
|
|
||||||
with pytest.raises(ValueError):
|
|
||||||
sg.load_allow(bad)
|
|
||||||
|
|
||||||
|
|
||||||
def test_block_and_warn(monkeypatch):
|
|
||||||
f = sg.cg.Finding("a.py", 1, "github token", "github token #abcd1234")
|
|
||||||
monkeypatch.setattr(sg, "MODE", "block")
|
|
||||||
assert sg.status_for([f], False)[0] == "failure"
|
|
||||||
assert sg.status_for([], False, grant=[f])[0] == "success"
|
|
||||||
monkeypatch.setattr(sg, "MODE", "warn")
|
|
||||||
state, desc, _ = sg.status_for([f], True)
|
|
||||||
assert state == "success" and desc.startswith("⚠ WARN (not blocking): 1 secret-shaped string in tree")
|
|
||||||
|
|
||||||
|
|
||||||
def test_public_scan_excuses_by_hash_and_by_path():
|
|
||||||
spec = importlib.util.spec_from_file_location("public_secret_scan", ROOT / "scripts" / "public_secret_scan.py")
|
|
||||||
ps = importlib.util.module_from_spec(spec)
|
|
||||||
spec.loader.exec_module(ps)
|
|
||||||
allow = {"windy-agent": {"hashes": {"aaaa1111"}, "paths": [("tests/keys/*", {"private key block"})]}}
|
|
||||||
assert ps.excused("windy-agent", "openai key", "aaaa1111", ["tests/x.py"], allow)
|
|
||||||
assert not ps.excused("windy-agent", "telegram bot token", "1354fc9b", ["tests/test_log_redaction.py"], allow)
|
|
||||||
assert ps.excused("windy-agent", "private key block", "ffff0000", ["tests/keys/a.pem"], allow)
|
|
||||||
# a PEM header anywhere outside the allowed paths still counts
|
|
||||||
assert not ps.excused("windy-agent", "private key block", "ffff0000", ["tests/keys/a.pem", "deploy/k.pem"], allow)
|
|
||||||
assert not ps.excused("other", "openai key", "aaaa1111", ["x"], allow)
|
|
||||||
|
|
||||||
|
|
||||||
HEX32 = "0123456789abcdef" * 2 # synthetic
|
|
||||||
|
|
||||||
|
|
||||||
def test_twilio_shapes_hash_only_and_no_md5_noise():
|
|
||||||
kinds = lambda t: [k for k, _ in ss.find(t)] # noqa: E731
|
|
||||||
assert kinds(f'TWILIO_AUTH_TOKEN = "{HEX32}"') == ["32-hex secret assignment"]
|
|
||||||
assert kinds(f"auth_token: {HEX32}") == ["32-hex secret assignment"]
|
|
||||||
assert kinds("AC" + HEX32) == ["twilio sid/api key"]
|
|
||||||
assert kinds("SK" + HEX32) == ["twilio sid/api key"]
|
|
||||||
# plain md5 / uuid-without-dashes / a 64-hex sha256 are NOT secrets by shape
|
|
||||||
assert kinds(f"md5 = {HEX32}") == []
|
|
||||||
assert kinds(f"checksum_key = {HEX32}{HEX32}") == []
|
|
||||||
assert kinds(f"name = 'x{HEX32}'") == []
|
|
||||||
# the hash is of the value alone, so renaming the variable keeps the same allow hash
|
|
||||||
a = ss.find(f"A_TOKEN={HEX32}")[0][1]
|
|
||||||
b = ss.find(f"OTHER_SECRET: '{HEX32}'")[0][1]
|
|
||||||
assert a == b == ss.h8(HEX32)
|
|
||||||
assert HEX32 not in repr(ss.find(f"A_TOKEN={HEX32}"))
|
|
||||||
|
|
||||||
|
|
||||||
def test_warn_kinds_do_not_block(monkeypatch):
|
|
||||||
f = sg.cg.Finding("a.py", 1, "32-hex secret assignment", "32-hex secret assignment #abcd1234")
|
|
||||||
monkeypatch.setattr(sg, "MODE", "block")
|
|
||||||
monkeypatch.setattr(sg, "WARN_KINDS", {"32-hex secret assignment"})
|
|
||||||
assert sg.status_for([f], True)[0] == "success"
|
|
||||||
monkeypatch.setattr(sg, "WARN_KINDS", set())
|
|
||||||
assert sg.status_for([f], True)[0] == "failure"
|
|
||||||
@@ -1,13 +0,0 @@
|
|||||||
# CI hygiene allow-list: installs that may float, or services that may publish
|
|
||||||
# a host port. House rule 6 (09-23): installs come from a lockfile. Every entry
|
|
||||||
# is an exception and MUST say why. Paths are fnmatch globs from the repo root.
|
|
||||||
# Owner: Windy Git lane (13); changes go through the orchestrator.
|
|
||||||
allow:
|
|
||||||
- repo: windy-pro
|
|
||||||
paths: [".github/workflows/ci.yml"]
|
|
||||||
# ONLY the old deploy job's two docker lines. That job is `if: false`
|
|
||||||
# (CD boundary, 2026-07), and the compose line runs ON windyword.ai inside
|
|
||||||
# the ssh string. Any other docker step in ci.yml still flags.
|
|
||||||
matches: ['docker build -f account-server/Dockerfile -t windy-pro:', 'docker compose down && docker compose up -d --build']
|
|
||||||
reason: "needs-docker false positive: the deploy job is if: false and its compose runs on the remote host over ssh. Added with the needs-docker rule (orchestrator option A, 09-23)."
|
|
||||||
|
|
||||||
@@ -35,13 +35,6 @@ allow:
|
|||||||
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
paths: ["src/client/web/src/pages/panels/MindPanel.jsx"]
|
||||||
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
reason: "Validates the USER's own OpenRouter key for BYOK (audit #10); spends no house money."
|
||||||
|
|
||||||
- repo: windy-pro
|
|
||||||
paths: ["src/client/web/src/pages/panels/MindKeychain.jsx"]
|
|
||||||
# ONLY these two endpoints: any other openrouter.ai call in this file (e.g.
|
|
||||||
# /api/v1/chat, i.e. inference) still flags. Orchestrator-approved 09-23.
|
|
||||||
matches: ['openrouter\.ai/auth\?', 'openrouter\.ai/api/v1/auth/keys']
|
|
||||||
reason: "BYOK key acquisition via OpenRouter OAuth PKCE; no inference; successor of MindPanel allow (ADR-064)."
|
|
||||||
|
|
||||||
- repo: windy-git
|
- repo: windy-git
|
||||||
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
paths: ["scripts/compute_guard.py", "ci/compute-guard-allow.yml"]
|
||||||
reason: "The guard's own pattern list and this file."
|
reason: "The guard's own pattern list and this file."
|
||||||
|
|||||||
@@ -1,19 +0,0 @@
|
|||||||
# Code Grant owns directly (orchestrator, 09-23): guard findings here are listed
|
|
||||||
# SEPARATELY in the guards status page and never hold up "block". Changes to
|
|
||||||
# these files are proposals for Grant / Windy Word 44, not a lane's fix.
|
|
||||||
grant_owned:
|
|
||||||
- repo: windy-pro
|
|
||||||
reason: "Windy Word desktop (Electron) + its release/installer builds: Grant's, built from the Mac mini."
|
|
||||||
paths:
|
|
||||||
- "src/client/desktop/*"
|
|
||||||
# ROOT .env.example only (exact path): documents the desktop app's BYOK dev
|
|
||||||
# fallback keys; the hub reads neither (8c, 09-23). account-server/.env.example
|
|
||||||
# is NOT matched and stays the hub lane's.
|
|
||||||
- ".env.example"
|
|
||||||
- "installer-v2/*"
|
|
||||||
- ".github/workflows/build-windows.yml"
|
|
||||||
- ".github/workflows/release-mac.yml"
|
|
||||||
- ".github/workflows/build-installer.yml"
|
|
||||||
- ".github/workflows/build-offline-installers.yml"
|
|
||||||
jobs:
|
|
||||||
".github/workflows/ci.yml": [reality-check, build-desktop, test-installer, build-electron]
|
|
||||||
@@ -1,59 +0,0 @@
|
|||||||
# Secret guard allow-list: KNOWN FAKE values that look like secrets (test
|
|
||||||
# fixtures, docs). Allowed BY HASH (sha256[:8] of the value), so a real secret
|
|
||||||
# in the same file still flags. Private-key blocks (the match is only the BEGIN
|
|
||||||
# line, same hash everywhere) are allowed by PATH + kind instead.
|
|
||||||
# Every entry MUST say why. Owner: Windy Git lane (13); changes via the orchestrator.
|
|
||||||
# Triage 09-24 (values never printed): each hash checked against every version of
|
|
||||||
# the lockbox; fakes judged by impossible length for the kind (real Anthropic keys
|
|
||||||
# ~108 chars, OpenAI 51 or 160+), fake-words, or identity with upstream public
|
|
||||||
# fixtures. NOT allowed, remove instead: 1354fc9b (old @Windy_0_bot token) and
|
|
||||||
# d49dc2ba (old Anthropic key), both real and revoked, in public windy-agent.
|
|
||||||
allow:
|
|
||||||
- repo: windy-code
|
|
||||||
hashes: [ac9265e5, 46eb1235]
|
|
||||||
reason: "Upstream microsoft/vscode terminalEnvironment.test.ts fixtures (identical hash upstream; public)."
|
|
||||||
- repo: windy-code
|
|
||||||
paths: ["build/azure-pipelines/common/publish.ts"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "Upstream VS Code build script (PEM header string in code, not a key)."
|
|
||||||
- repo: windy-agent
|
|
||||||
hashes: [a9235a6d, dd6a2baa, 02c362d8, a6f6ff79, f7503b21, d0c94833, 4ab092e3, e3aa1eb8, 833382ee]
|
|
||||||
reason: "Redaction/sanitizer test fixtures; lengths impossible for real Anthropic/OpenAI keys; never in the lockbox."
|
|
||||||
- repo: windy-agent
|
|
||||||
hashes: [89bd408f, b8c94b72, bf23cdf5, d1d85dfe, e3e07f06]
|
|
||||||
reason: "09-24 #395 replacement fixtures (each contains FAKE; token-SHAPED on purpose so redaction tests prove real tokens are scrubbed); verified by Windy Agent sha-for-sha against every lockbox version: never real. Weekly scan 09-28."
|
|
||||||
- repo: windy-agent
|
|
||||||
paths: ["tests/test_agent_keys.py"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "Test-generated key material for agent-key tests."
|
|
||||||
- repo: windy-mind
|
|
||||||
hashes: [f8a630b2]
|
|
||||||
reason: "Provider test fixture (27 chars; a real Anthropic key is ~108)."
|
|
||||||
- repo: windy-pro
|
|
||||||
hashes: [756de8d8, 7828319d, 1a5d44a2]
|
|
||||||
reason: ".env.production.example placeholder + crash-summary test fixtures (AWS doc EXAMPLE key shape, short fake Slack token)."
|
|
||||||
- repo: windy-pro
|
|
||||||
paths: ["account-server/docs/oauth-providers.md"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "Docs show the PEM header format; no key material."
|
|
||||||
- repo: windytalk
|
|
||||||
hashes: [baf8656a]
|
|
||||||
reason: "Diagnostics redaction test fixture (fake-word in value)."
|
|
||||||
- repo: eternitas
|
|
||||||
paths: ["tests/golden_vectors/**", "tests/test_soul_vault_key_separation.py"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "Test vectors and throwaway keys for signature/vault tests."
|
|
||||||
- repo: windy-drops
|
|
||||||
paths: ["tools/conformance/test-keys/*"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "Conformance-suite test keys (named test-private.pem)."
|
|
||||||
- repo: windy-git
|
|
||||||
paths: ["api/tests/test_secret_guard.py"]
|
|
||||||
kinds: [private key block]
|
|
||||||
reason: "The guard's own test uses a PEM header string as a sample."
|
|
||||||
- repo: windy-code
|
|
||||||
hashes: ["23f32607"]
|
|
||||||
reason: "VS Code OSS extensions' package.json aiKey: Microsoft's public telemetry (App Insights) key, shipped in every VS Code build; not a Windy credential."
|
|
||||||
- repo: windytalk
|
|
||||||
hashes: ["c4189d79"]
|
|
||||||
reason: "apps/desktop/test/diagnostics.test.ts redaction fixture (hexSecret beside a fake sk-ant token); hash checked against the lockbox 10-01: not present."
|
|
||||||
@@ -7,7 +7,6 @@
|
|||||||
<div class="wg-cta">
|
<div class="wg-cta">
|
||||||
{{if not .IsSigned}}
|
{{if not .IsSigned}}
|
||||||
<a class="ui primary button" href="{{AppSubUrl}}/user/login">Sign in with Windy</a>
|
<a class="ui primary button" href="{{AppSubUrl}}/user/login">Sign in with Windy</a>
|
||||||
<p class="wg-note" style="margin-top:.9rem;font-size:.95rem;opacity:.75">Invite only at launch: new accounts are not open yet.</p>
|
|
||||||
{{else}}
|
{{else}}
|
||||||
<a class="ui primary button" href="{{AppSubUrl}}/{{.SignedUser.Name}}?tab=repositories">Your repositories</a>
|
<a class="ui primary button" href="{{AppSubUrl}}/{{.SignedUser.Name}}?tab=repositories">Your repositories</a>
|
||||||
{{end}}
|
{{end}}
|
||||||
@@ -19,8 +18,8 @@
|
|||||||
<p>Git and LFS over Windy Cloud storage. Source, weights and adapters live side by side.</p></div>
|
<p>Git and LFS over Windy Cloud storage. Source, weights and adapters live side by side.</p></div>
|
||||||
<div class="wg-card"><h3>Agents are first-class</h3>
|
<div class="wg-card"><h3>Agents are first-class</h3>
|
||||||
<p>An agent signs in with its own Eternitas passport — not a human's borrowed token — and its work is attributable to it.</p></div>
|
<p>An agent signs in with its own Eternitas passport — not a human's borrowed token — and its work is attributable to it.</p></div>
|
||||||
<div class="wg-card"><h3>Invite only, for now</h3>
|
<div class="wg-card"><h3>Your account, no second password</h3>
|
||||||
<p>Windy Git accounts are by invitation while we launch. If you've been invited, you sign in with your Windy account, with no second password.</p></div>
|
<p>Sign in with the Windy account you already have. Nothing new to remember.</p></div>
|
||||||
<div class="wg-card"><h3>Kept, and kept elsewhere</h3>
|
<div class="wg-card"><h3>Kept, and kept elsewhere</h3>
|
||||||
<p>Every repository is bundled nightly to off-site storage, and the restore is rehearsed rather than assumed.</p></div>
|
<p>Every repository is bundled nightly to off-site storage, and the restore is rehearsed rather than assumed.</p></div>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -1,61 +0,0 @@
|
|||||||
{{/* Windy Git override of Gitea 1.24.6 templates/user/auth/link_account.tmpl.
|
|
||||||
Forge registration is CLOSED at launch (Grant, 09-23), so a Windy account
|
|
||||||
with no forge account used to land on Gitea's raw "Registration is disabled"
|
|
||||||
error. That case now gets a plain invite-only page; every other case is
|
|
||||||
Gitea's template unchanged (re-diff it on Gitea upgrades). No change to who
|
|
||||||
can register. */}}
|
|
||||||
{{if and .DisableRegistration (not .user_exists)}}
|
|
||||||
{{template "base/head" .}}
|
|
||||||
<div role="main" aria-label="Windy Git is invite-only" class="page-content user link-account">
|
|
||||||
<div class="ui middle very relaxed page grid">
|
|
||||||
<div class="column tw-my-5">
|
|
||||||
<div class="tw-flex tw-flex-col tw-gap-4 tw-max-w-2xl tw-m-auto">
|
|
||||||
<h4 class="ui top attached header center">Windy Git is invite-only while we launch</h4>
|
|
||||||
<div class="ui attached segment">
|
|
||||||
<p>You're signed in with your Windy account{{if or .user_name .email}} as <strong>{{or .user_name .email}}</strong>{{end}}. Nothing is wrong with it: Windy Git just isn't open to every account yet.</p>
|
|
||||||
<p>We'll let you know when it opens.</p>
|
|
||||||
<a class="ui primary button" href="https://app.windyword.ai/dashboard">Back to your dashboard</a>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
{{template "base/footer" .}}
|
|
||||||
{{else}}
|
|
||||||
{{template "base/head" .}}
|
|
||||||
<div role="main" aria-label="{{.Title}}" class="page-content user link-account">
|
|
||||||
<overflow-menu class="ui secondary pointing tabular top attached borderless menu secondary-nav">
|
|
||||||
<div class="overflow-menu-items tw-justify-center">
|
|
||||||
<!-- TODO handle .ShowRegistrationButton once other login bugs are fixed -->
|
|
||||||
{{if not .AllowOnlyInternalRegistration}}
|
|
||||||
<a class="item {{if not .user_exists}}active{{end}}"
|
|
||||||
data-tab="auth-link-signup-tab">
|
|
||||||
{{ctx.Locale.Tr "auth.oauth_signup_tab"}}
|
|
||||||
</a>
|
|
||||||
{{end}}
|
|
||||||
<a class="item {{if .user_exists}}active{{end}}"
|
|
||||||
data-tab="auth-link-signin-tab">
|
|
||||||
{{ctx.Locale.Tr "auth.oauth_signin_tab"}}
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
</overflow-menu>
|
|
||||||
<div class="ui middle very relaxed page grid">
|
|
||||||
<div class="column tw-my-5">
|
|
||||||
{{/* these styles are quite tricky but it needs to be the same as the signin page */}}
|
|
||||||
<div class="ui tab {{if not .user_exists}}active{{end}}" data-tab="auth-link-signup-tab">
|
|
||||||
<div class="tw-flex tw-flex-col tw-gap-4 tw-max-w-2xl tw-m-auto">
|
|
||||||
{{if .AutoRegistrationFailedPrompt}}<div class="ui message">{{.AutoRegistrationFailedPrompt}}</div>{{end}}
|
|
||||||
{{template "user/auth/signup_inner" .}}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
<div class="ui tab {{if .user_exists}}active{{end}}" data-tab="auth-link-signin-tab">
|
|
||||||
<div class="tw-flex tw-flex-col tw-gap-4 tw-max-w-2xl tw-m-auto">
|
|
||||||
{{template "user/auth/signin_inner" .}}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{{template "base/footer" .}}
|
|
||||||
{{end}}
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
[Service]
|
|
||||||
# Orchestrator 09-23: compute-guard BLOCKS lane-owned findings; Grant-owned (ci/grant-owned.yml) stay WARN.
|
|
||||||
Environment=COMPUTE_GUARD_MODE=block
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
[Service]
|
|
||||||
# Orchestrator 09-24: secret-guard BLOCKS lane-owned findings; Grant-owned stay WARN.
|
|
||||||
Environment=SECRET_GUARD_MODE=block
|
|
||||||
@@ -110,12 +110,7 @@ their CI permanently, not a stopgap:
|
|||||||
- **Image-build jobs** (name matches `docker`) post nothing: job containers
|
- **Image-build jobs** (name matches `docker`) post nothing: job containers
|
||||||
have no Docker daemon by design (I-5), so they are red on every commit. A
|
have no Docker daemon by design (I-5), so they are red on every commit. A
|
||||||
rootless builder (BuildKit rootless / buildx in the capped dind) is the open
|
rootless builder (BuildKit rootless / buildx in the capped dind) is the open
|
||||||
decision that would bring them back. Jobs that need Docker but are named otherwise go in
|
decision that would bring them back.
|
||||||
`BRIDGE_NO_DAEMON` (empty since eternitas converted to ci/smoke, #179). DECIDED 09-23 (orchestrator,
|
|
||||||
option A): lanes convert these jobs to no-Docker smoke tests (job `services:` +
|
|
||||||
start the app + curl /health); the real image build is the deploy step on the
|
|
||||||
target host. ci-hygiene flags docker build/compose/run in workflows ("needs docker").
|
|
||||||
No host Docker socket for CI without a separate decision.
|
|
||||||
|
|
||||||
**Onboarding another private repo** — the promotion steps below, then:
|
**Onboarding another private repo** — the promotion steps below, then:
|
||||||
|
|
||||||
|
|||||||
@@ -1,258 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""CI hygiene guard: installs come from a lockfile, never "latest" (house rule 6).
|
|
||||||
|
|
||||||
A floating install lets CI test different versions than prod ships, and a
|
|
||||||
rebuild silently changes prod. Windy Cloud's OpenAPI test failed on exactly
|
|
||||||
that (fastapi 0.141.1 in CI vs 0.136.0 on the dev box) and all three Cloud
|
|
||||||
cells floated in prod. Also flags services that publish a HOST port: every
|
|
||||||
CI job shares one dind daemon, so two jobs publishing 5432 collide ("port is
|
|
||||||
already allocated", Windy Mind runs 147/176).
|
|
||||||
|
|
||||||
WARN-ONLY (`windy-git/ci-hygiene`, green + "⚠ WARN"); CI_HYGIENE_MODE=block
|
|
||||||
turns it red once the lanes report clean. Scans CI workflow files and
|
|
||||||
Dockerfiles only. PR heads: lines the PR adds. Default branch: every line.
|
|
||||||
|
|
||||||
OK (not flagged):
|
|
||||||
pip / uv pip install -r FILE (with or without --require-hashes), --no-deps,
|
|
||||||
exact pins (tool==1.2.3), pip/setuptools/wheel upgrades
|
|
||||||
uv sync --locked | --frozen npm ci
|
|
||||||
npm install pkg@1.2.3 (every package exact-pinned)
|
|
||||||
yarn install --frozen-lockfile / --immutable pnpm install --frozen-lockfile
|
|
||||||
Also flagged: `:latest` images (FROM / COPY --from / image: / docker://) and
|
|
||||||
`COPY uv.lock* ...`-style globs that build without the lock (Windy Mail #147).
|
|
||||||
Exceptions: ci/ci-hygiene-allow.yml, one reason per entry.
|
|
||||||
|
|
||||||
python3 scripts/ci_hygiene.py report [repo ...]
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import shlex
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
||||||
import compute_guard as cg # noqa: E402 (shared walker, cache and allow-list loader)
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
ALLOW_FILE = Path(os.environ.get("CI_HYGIENE_ALLOW", ROOT / "ci" / "ci-hygiene-allow.yml"))
|
|
||||||
MODE = os.environ.get("CI_HYGIENE_MODE", "warn")
|
|
||||||
|
|
||||||
# CI workflow files and Dockerfiles; never vendored copies.
|
|
||||||
INCLUDE = re.compile(r"(^|/)\.(github|gitea)/workflows/[^/]+\.ya?ml$|(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
|
||||||
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
|
||||||
PREFILTER = (r"pip3? install|pip install|uv sync|npm (install|i )|yarn install|pnpm install"
|
|
||||||
r"|^\s*-\s*['\"]?[0-9]+:[0-9]+|:latest|lock[^ ]*\*"
|
|
||||||
r"|docker[ -]compose|docker (build|buildx|run)|docker/build-push-action")
|
|
||||||
|
|
||||||
TOOLING = {"pip", "setuptools", "wheel"}
|
|
||||||
NO_DOCKER_FIX = "use job services: + a no-Docker smoke test; the image builds at deploy"
|
|
||||||
DOCKER_FILE = re.compile(r"(^|/)(Dockerfile[^/]*|[^/]+\.Dockerfile)$")
|
|
||||||
LATEST = re.compile(r"(?:^\s*FROM\s+(?:--platform=\S+\s+)?|--from=|image:\s*['\"]?|docker://)([\w./-]+):latest\b", re.I)
|
|
||||||
LOCKNAME = re.compile(r"(uv\.lock|poetry\.lock|package-lock\.json|pnpm-lock\.yaml|yarn\.lock|requirements[^ ]*\.(txt|lock))", re.I)
|
|
||||||
EXACT_PY = re.compile(r"^[A-Za-z0-9._-]+(\[[^\]]*\])?==[A-Za-z0-9.+!-]+$")
|
|
||||||
EXACT_NPM = re.compile(r"^(@[^/@]+/)?[^/@]+@\d+\.\d+\.\d+([-+][0-9A-Za-z.-]+)?$")
|
|
||||||
HOST_PORT = re.compile(r"^\s*-\s*['\"]?(\d{2,5}):(\d{2,5})['\"]?\s*(#.*)?$")
|
|
||||||
PIP_VALUE_FLAGS = {"-c", "--constraint", "-i", "--index-url", "--extra-index-url", "-f",
|
|
||||||
"--find-links", "--target", "-t", "--python", "--prefix", "--root", "--platform",
|
|
||||||
"--python-version", "--implementation", "--abi", "--only-binary", "--no-binary"}
|
|
||||||
|
|
||||||
|
|
||||||
def path_ok(path: str) -> bool:
|
|
||||||
return bool(INCLUDE.search(path)) and not NEVER.search(path)
|
|
||||||
|
|
||||||
|
|
||||||
def _commands(text: str) -> list[list[str]]:
|
|
||||||
"""Split a shell line into simple commands (&&, ||, ;, |), tokenized."""
|
|
||||||
out = []
|
|
||||||
for part in re.split(r"&&|\|\||;|\|", text):
|
|
||||||
try:
|
|
||||||
toks = shlex.split(part, comments=True)
|
|
||||||
except ValueError:
|
|
||||||
toks = part.split()
|
|
||||||
# Dockerfile RUN prefix / sudo / env-prefixed assignments
|
|
||||||
while toks and (toks[0] in ("RUN", "sudo", "exec", "-", "run:", "command:")
|
|
||||||
or re.match(r"^[A-Z_][A-Z0-9_]*=", toks[0])):
|
|
||||||
toks = toks[1:]
|
|
||||||
if toks:
|
|
||||||
out.append(toks)
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def _pip_problem(args: list[str]) -> str | None:
|
|
||||||
if "-r" in args or "--requirement" in args or any(a.startswith("--requirement=") for a in args):
|
|
||||||
return None
|
|
||||||
if "--no-deps" in args:
|
|
||||||
return None
|
|
||||||
pkgs, skip = [], False
|
|
||||||
for a in args:
|
|
||||||
if skip:
|
|
||||||
skip = False
|
|
||||||
continue
|
|
||||||
if a in PIP_VALUE_FLAGS:
|
|
||||||
skip = True
|
|
||||||
continue
|
|
||||||
if a.startswith("-") and a not in ("-e", "--editable"):
|
|
||||||
continue
|
|
||||||
if a in ("-e", "--editable"):
|
|
||||||
continue
|
|
||||||
pkgs.append(a)
|
|
||||||
loose = [p for p in pkgs if not EXACT_PY.match(p) and p.split("[")[0].lower() not in TOOLING]
|
|
||||||
if loose:
|
|
||||||
return f"floating pip install: {' '.join(loose)[:40]}"
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
|
||||||
if cg.COMMENT.match(text):
|
|
||||||
return []
|
|
||||||
hits = []
|
|
||||||
if "/workflows/" in path and HOST_PORT.match(text):
|
|
||||||
hits.append(("host port", f"service publishes host port {HOST_PORT.match(text).group(1)} (shared dind)"))
|
|
||||||
return hits
|
|
||||||
# Windy Mail #147: a `:latest` build/tool image floats exactly like an
|
|
||||||
# unpinned package, and `COPY uv.lock* ./` builds WITHOUT the lock when it
|
|
||||||
# is missing instead of failing.
|
|
||||||
m = LATEST.search(text)
|
|
||||||
if m:
|
|
||||||
hits.append(("floating image", f"{m.group(1)}:latest"))
|
|
||||||
if DOCKER_FILE.search(path) and re.match(r"^\s*COPY\b", text, re.I):
|
|
||||||
globbed = [t for t in text.split() if "*" in t and LOCKNAME.search(t)]
|
|
||||||
if globbed:
|
|
||||||
hits.append(("optional lock", f"COPY {globbed[0]} (must fail if the lock is missing)"))
|
|
||||||
# Windy Git jobs get NO Docker daemon (I-5), so a docker build/compose/run
|
|
||||||
# step in CI can never pass here (orchestrator 09-23, option A). The real
|
|
||||||
# image build is the deploy step on the target host.
|
|
||||||
if "/workflows/" in path and re.search(r"uses:\s*['\"]?docker/build-push-action", text):
|
|
||||||
hits.append(("needs docker", "docker/build-push-action in CI (no Docker daemon on Windy Git; " + NO_DOCKER_FIX + ")"))
|
|
||||||
for toks in _commands(text):
|
|
||||||
low = [t.lower() for t in toks]
|
|
||||||
if "/workflows/" in path and (
|
|
||||||
low[:2] in (["docker", "build"], ["docker", "buildx"], ["docker", "run"], ["docker", "compose"])
|
|
||||||
or low[:1] == ["docker-compose"]
|
|
||||||
):
|
|
||||||
hits.append(("needs docker", f"{' '.join(low[:2])} in CI (no Docker daemon on Windy Git; " + NO_DOCKER_FIX + ")"))
|
|
||||||
continue
|
|
||||||
# pip install / python -m pip install / uv pip install
|
|
||||||
for i in range(len(low) - 1):
|
|
||||||
if os.path.basename(low[i]) in ("pip", "pip3") and low[i + 1] == "install":
|
|
||||||
prob = _pip_problem(toks[i + 2:])
|
|
||||||
if prob:
|
|
||||||
hits.append(("floating install", prob))
|
|
||||||
break
|
|
||||||
if low[:2] == ["uv", "sync"] and not ({"--locked", "--frozen"} & set(low)):
|
|
||||||
hits.append(("floating install", "uv sync without --locked/--frozen"))
|
|
||||||
if low[:1] == ["npm"] and len(low) > 1 and low[1] in ("install", "i", "add"):
|
|
||||||
pkgs = [t for t in toks[2:] if not t.startswith("-")]
|
|
||||||
if not pkgs or not all(EXACT_NPM.match(p) for p in pkgs):
|
|
||||||
hits.append(("floating install", f"npm {low[1]} {' '.join(pkgs)[:30]}".strip() + " (use npm ci)"))
|
|
||||||
if low[:2] == ["yarn", "install"] and not ({"--frozen-lockfile", "--immutable"} & set(low)):
|
|
||||||
hits.append(("floating install", "yarn install without --frozen-lockfile"))
|
|
||||||
if low[:2] == ["pnpm", "install"] and "--frozen-lockfile" not in low:
|
|
||||||
hits.append(("floating install", "pnpm install without --frozen-lockfile"))
|
|
||||||
return hits
|
|
||||||
|
|
||||||
|
|
||||||
_DISABLED: dict[str, set[str]] | None = None
|
|
||||||
|
|
||||||
|
|
||||||
def disabled_workflows() -> dict[str, set[str]]:
|
|
||||||
"""Workflow file names Gitea has DISABLED per repo (lowercased repo name).
|
|
||||||
|
|
||||||
Deploy/release workflows are disabled on Windy Git: they run on the target
|
|
||||||
host, where a Docker daemon really exists, so "needs docker" must not flag
|
|
||||||
them. One bounded query per process; on any failure nothing is excused
|
|
||||||
(flag rather than hide).
|
|
||||||
"""
|
|
||||||
global _DISABLED
|
|
||||||
if _DISABLED is not None:
|
|
||||||
return _DISABLED
|
|
||||||
_DISABLED = {}
|
|
||||||
query = ("select coalesce(json_object_agg(r.lower_name, u.config::json->'DisabledWorkflows'), '{}'::json)"
|
|
||||||
" from repo_unit u join repository r on r.id = u.repo_id"
|
|
||||||
" where u.type = 10 and u.config like '%DisabledWorkflows%';")
|
|
||||||
try:
|
|
||||||
out = subprocess.run(
|
|
||||||
["docker", "exec", "-i", "windy-git-db-1", "sh", "-c",
|
|
||||||
'psql -U "$POSTGRES_USER" -d gitea -At -v ON_ERROR_STOP=1'],
|
|
||||||
input=query, capture_output=True, text=True, check=True, timeout=30,
|
|
||||||
).stdout.strip()
|
|
||||||
_DISABLED = {k: set(v or []) for k, v in json.loads(out or "{}").items()}
|
|
||||||
except (subprocess.SubprocessError, OSError, ValueError):
|
|
||||||
pass
|
|
||||||
return _DISABLED
|
|
||||||
|
|
||||||
|
|
||||||
def _runs_here(repo: str, findings):
|
|
||||||
"""Drop "needs docker" hits in workflows that never run on Windy Git."""
|
|
||||||
if findings is None:
|
|
||||||
return None
|
|
||||||
off = disabled_workflows().get(repo.lower(), set())
|
|
||||||
return [f for f in findings if not (f.kind == "needs docker" and Path(f.path).name in off)]
|
|
||||||
|
|
||||||
|
|
||||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
|
||||||
return _runs_here(repo, _check(repo, sha, default_branch, is_default_head))
|
|
||||||
|
|
||||||
|
|
||||||
def _check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
|
||||||
return None
|
|
||||||
allow = cg.load_allow(ALLOW_FILE)
|
|
||||||
rules = hashlib.sha256((PREFILTER + INCLUDE.pattern + EXACT_PY.pattern + EXACT_NPM.pattern).encode()).hexdigest()[:8]
|
|
||||||
fp = cg._fingerprint(allow) + ":" + rules # hashlib, not hash(): hash() is per-process random
|
|
||||||
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
|
||||||
if is_default_head:
|
|
||||||
return cg.cached_scan(f"hyg-tree:{repo}:{sha}:{fp}",
|
|
||||||
lambda: cg.scan_tree(repo, bare, sha, allow, prefilter=PREFILTER, **kw))
|
|
||||||
return cg.cached_scan(f"hyg-pr:{repo}:{sha}:{fp}",
|
|
||||||
lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, allow, **kw))
|
|
||||||
|
|
||||||
|
|
||||||
def status_for(findings, whole_tree: bool, grant=()):
|
|
||||||
"""Same contract as compute_guard.status_for: `grant` findings never block."""
|
|
||||||
scope = "in CI/Dockerfiles" if whole_tree else "added"
|
|
||||||
if not findings and grant:
|
|
||||||
g, n = grant[0], len(grant)
|
|
||||||
desc = f"⚠ WARN (Grant-owned, not blocking): {n} CI hygiene issue{'s' if n > 1 else ''} {scope}, e.g. {g.path}:{g.line} {g.match}"
|
|
||||||
return "success", desc[:140], g
|
|
||||||
if not findings:
|
|
||||||
return "success", f"OK: no floating install or host-port service {scope}", None
|
|
||||||
f = findings[0]
|
|
||||||
n = len(findings)
|
|
||||||
state = "failure" if MODE == "block" else "success"
|
|
||||||
lead = "BLOCKED" if MODE == "block" else "⚠ WARN (not blocking)"
|
|
||||||
return state, f"{lead}: {n} CI hygiene issue{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
|
||||||
|
|
||||||
|
|
||||||
def report(repos: list[str]) -> int:
|
|
||||||
allow = cg.load_allow(ALLOW_FILE)
|
|
||||||
total = 0
|
|
||||||
for repo in repos:
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
if not bare.is_dir():
|
|
||||||
print(f"## {repo}: no sync clone, skipped")
|
|
||||||
continue
|
|
||||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
|
||||||
sha = cg._git(bare, "rev-parse", head).strip()
|
|
||||||
fs = _runs_here(repo, cg.scan_tree(repo, bare, sha, allow, line_fn=scan_line, path_ok=path_ok, prefilter=PREFILTER))
|
|
||||||
total += len(fs)
|
|
||||||
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} issue(s)")
|
|
||||||
for f in fs:
|
|
||||||
print(f" {f.path}:{f.line} [{f.kind}] {f.match}")
|
|
||||||
print(f"TOTAL {total}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
|
||||||
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
|
||||||
p.name.removesuffix(".git") for p in cg.WORK.glob("*.git"))
|
|
||||||
sys.exit(report(sys.argv[2:] or default))
|
|
||||||
sys.exit(__doc__)
|
|
||||||
@@ -98,18 +98,9 @@ def load_allow(path: Path = ALLOW_FILE) -> list[dict]:
|
|||||||
return entries
|
return entries
|
||||||
|
|
||||||
|
|
||||||
def allowed(repo: str, path: str, allow: list[dict], text: str | None = None) -> bool:
|
def allowed(repo: str, path: str, allow: list[dict]) -> bool:
|
||||||
"""An entry may carry `matches:` (regexes): then only lines matching one of
|
|
||||||
them are allowed, so an allowed file can't smuggle in a NEW call (e.g. an
|
|
||||||
OAuth sign-in endpoint is allowed, an inference endpoint in the same file
|
|
||||||
still flags). Entries without `matches` cover the whole path."""
|
|
||||||
for e in allow:
|
for e in allow:
|
||||||
if e["repo"] != repo or not any(fnmatch.fnmatch(path, g) for g in e["paths"]):
|
if e["repo"] == repo and any(fnmatch.fnmatch(path, g) for g in e["paths"]):
|
||||||
continue
|
|
||||||
pats = e.get("matches")
|
|
||||||
if not pats:
|
|
||||||
return True
|
|
||||||
if text is not None and any(re.search(rx, text) for rx in pats):
|
|
||||||
return True
|
return True
|
||||||
return False
|
return False
|
||||||
|
|
||||||
@@ -139,20 +130,11 @@ def _git(bare: Path, *args: str) -> str:
|
|||||||
).stdout
|
).stdout
|
||||||
|
|
||||||
|
|
||||||
def _default_path_ok(path: str) -> bool:
|
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict]) -> list[Finding]:
|
||||||
return not SKIP.search(path)
|
|
||||||
|
|
||||||
|
|
||||||
def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict], *, line_fn=None,
|
|
||||||
path_ok=None, prefilter: str | None = None) -> list[Finding]:
|
|
||||||
"""Every line in the tree at `sha` (default branch: the baseline)."""
|
"""Every line in the tree at `sha` (default branch: the baseline)."""
|
||||||
# A cheap prefilter by git, then the real rules in Python.
|
# A cheap prefilter by git, then the real rules in Python.
|
||||||
# Other guards (ci_hygiene) reuse this walker with their own line rules.
|
pre = "|".join([re.escape(h) for h in HOSTS] + KEYS + ["anthropic", "openai", "groq", "mistral",
|
||||||
line_fn = line_fn or scan_line
|
"generativeai", "genai", "cohere", "together", "cerebras", "litellm"])
|
||||||
path_ok = path_ok or _default_path_ok
|
|
||||||
pre = prefilter or "|".join([re.escape(h) for h in HOSTS] + KEYS + [
|
|
||||||
"anthropic", "openai", "groq", "mistral", "generativeai", "genai", "cohere",
|
|
||||||
"together", "cerebras", "litellm"])
|
|
||||||
try:
|
try:
|
||||||
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
out = _git(bare, "grep", "-nIE", "-e", pre, sha, "--", ".")
|
||||||
except subprocess.CalledProcessError as e:
|
except subprocess.CalledProcessError as e:
|
||||||
@@ -166,26 +148,24 @@ def scan_tree(repo: str, bare: Path, sha: str, allow: list[dict], *, line_fn=Non
|
|||||||
_, path, line, text = raw.split(":", 3)
|
_, path, line, text = raw.split(":", 3)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
if not path_ok(path) or allowed(repo, path, allow, text):
|
if SKIP.search(path) or allowed(repo, path, allow):
|
||||||
continue
|
continue
|
||||||
for kind, match in line_fn(path, text):
|
for kind, match in scan_line(path, text):
|
||||||
found.append(Finding(path, int(line), kind, match))
|
found.append(Finding(path, int(line), kind, match))
|
||||||
return found
|
return found
|
||||||
|
|
||||||
|
|
||||||
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict], **kw) -> list[Finding]:
|
def scan_added(repo: str, bare: Path, base_ref: str, sha: str, allow: list[dict]) -> list[Finding]:
|
||||||
"""Only the lines a PR adds, vs its merge-base with the default branch."""
|
"""Only the lines a PR adds, vs its merge-base with the default branch."""
|
||||||
mb = _git(bare, "merge-base", base_ref, sha).strip()
|
mb = _git(bare, "merge-base", base_ref, sha).strip()
|
||||||
diff = _git(bare, "diff", "-U0", "--no-color", "--no-ext-diff", mb, sha)
|
diff = _git(bare, "diff", "-U0", "--no-color", "--no-ext-diff", mb, sha)
|
||||||
return parse_added(repo, diff, allow, **kw)
|
return parse_added(repo, diff, allow)
|
||||||
|
|
||||||
|
|
||||||
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,\d+)? @@")
|
||||||
|
|
||||||
|
|
||||||
def parse_added(repo: str, diff: str, allow: list[dict], *, line_fn=None, path_ok=None) -> list[Finding]:
|
def parse_added(repo: str, diff: str, allow: list[dict]) -> list[Finding]:
|
||||||
line_fn = line_fn or scan_line
|
|
||||||
path_ok = path_ok or _default_path_ok
|
|
||||||
found, path, line = [], None, 0
|
found, path, line = [], None, 0
|
||||||
for raw in diff.splitlines():
|
for raw in diff.splitlines():
|
||||||
if raw.startswith("+++ "):
|
if raw.startswith("+++ "):
|
||||||
@@ -199,8 +179,8 @@ def parse_added(repo: str, diff: str, allow: list[dict], *, line_fn=None, path_o
|
|||||||
if path is None or raw.startswith("--- "):
|
if path is None or raw.startswith("--- "):
|
||||||
continue
|
continue
|
||||||
if raw.startswith("+"):
|
if raw.startswith("+"):
|
||||||
if path_ok(path) and not allowed(repo, path, allow, raw[1:]):
|
if not (SKIP.search(path) or allowed(repo, path, allow)):
|
||||||
for kind, match in line_fn(path, raw[1:]):
|
for kind, match in scan_line(path, raw[1:]):
|
||||||
found.append(Finding(path, line, kind, match))
|
found.append(Finding(path, line, kind, match))
|
||||||
line += 1
|
line += 1
|
||||||
return found
|
return found
|
||||||
@@ -234,21 +214,10 @@ def cached_scan(key: str, fn) -> list[Finding]:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def fetched(bare: Path, sha: str) -> bool:
|
|
||||||
"""Is `sha` in the sync clone yet? The bridge learns PR / default heads from
|
|
||||||
GitHub's API AFTER the sync fetched, so a push in between is simply not here
|
|
||||||
until the next 5-min cycle. That is a race, not an error: skip quietly."""
|
|
||||||
try:
|
|
||||||
_git(bare, "cat-file", "-e", f"{sha}^{{commit}}")
|
|
||||||
return True
|
|
||||||
except subprocess.CalledProcessError:
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> list[Finding] | None:
|
def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> list[Finding] | None:
|
||||||
"""Findings for one commit, or None when the guard can't run (never a fake OK)."""
|
"""Findings for one commit, or None when the guard can't run (never a fake OK)."""
|
||||||
bare = WORK / f"{repo}.git"
|
bare = WORK / f"{repo}.git"
|
||||||
if not bare.is_dir() or not fetched(bare, sha):
|
if not bare.is_dir():
|
||||||
return None
|
return None
|
||||||
allow = load_allow()
|
allow = load_allow()
|
||||||
fp = _fingerprint(allow)
|
fp = _fingerprint(allow)
|
||||||
@@ -260,19 +229,9 @@ def check(repo: str, sha: str, default_branch: str, is_default_head: bool) -> li
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def status_for(findings: list[Finding], whole_tree: bool,
|
def status_for(findings: list[Finding], whole_tree: bool) -> tuple[str, str, Finding | None]:
|
||||||
grant: list[Finding] = ()) -> tuple[str, str, Finding | None]:
|
"""(state, description, first finding) for the GitHub commit status."""
|
||||||
"""(state, description, first finding) for the GitHub commit status.
|
|
||||||
|
|
||||||
`findings` = lane-owned (these block in MODE=block); `grant` = findings in
|
|
||||||
Grant-owned code (ci/grant-owned.yml): always WARN, never red (orchestrator
|
|
||||||
09-23: his desktop work is never blocked by us)."""
|
|
||||||
scope = "in tree" if whole_tree else "added"
|
scope = "in tree" if whole_tree else "added"
|
||||||
if not findings and grant:
|
|
||||||
g, n = grant[0], len(grant)
|
|
||||||
desc = (f"⚠ WARN (Grant-owned, not blocking): {n} direct AI-provider use{'s' if n > 1 else ''} "
|
|
||||||
f"{scope}, e.g. {g.path}:{g.line} {g.match}")
|
|
||||||
return "success", desc[:140], g
|
|
||||||
if not findings:
|
if not findings:
|
||||||
what = "no direct AI-provider use in tree" if whole_tree else "no direct AI-provider use added"
|
what = "no direct AI-provider use in tree" if whole_tree else "no direct AI-provider use added"
|
||||||
return "success", f"OK: {what} (Windy Mind is the only door)", None
|
return "success", f"OK: {what} (Windy Mind is the only door)", None
|
||||||
|
|||||||
@@ -1,161 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Live status of the repo guards (compute-guard + ci-hygiene + secret-guard) as one markdown page.
|
|
||||||
|
|
||||||
Scans every bridged repo's DEFAULT branch with both guards and renders what is
|
|
||||||
left, per repo and owner lane. Findings in code Grant owns (ci/grant-owned.yml:
|
|
||||||
windy-pro's desktop app and its build jobs) are listed in their OWN section and
|
|
||||||
do not count against "ready to block": those are proposals for Grant, not a
|
|
||||||
lane's fix (orchestrator, 09-23).
|
|
||||||
|
|
||||||
sudo python3 scripts/guards_report.py > GUARDS_STATUS.md
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import fnmatch
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
import time
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
||||||
import ci_hygiene as hy # noqa: E402
|
|
||||||
import compute_guard as cg # noqa: E402
|
|
||||||
import secret_guard as sgd # noqa: E402
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
OWNED = Path(os.environ.get("GRANT_OWNED", ROOT / "ci" / "grant-owned.yml"))
|
|
||||||
REPOS = os.environ.get("BRIDGE_REPOS", "").split() or [
|
|
||||||
"windy-chat", "windy-mail", "windy-calendar", "Windy-Clone", "WindyCloud", "windy-search",
|
|
||||||
"windy-connect", "windy-drops", "windy-code-web", "windy-code", "windy-traveler",
|
|
||||||
"windy-registry", "eternitas", "windy-translate", "windytranslate-site", "windytraveler-site",
|
|
||||||
"windy-hand", "windy-cloud-sites", "windy-cloud-domains", "windy-cloud-vps", "windytalk",
|
|
||||||
"windy-pro", "windy-mind", "windy-git", "windy-inbox"]
|
|
||||||
# Owner lane per repo = the session name to message (orchestrator routing, 09-23 ~22:45Z:
|
|
||||||
# hub/account-server/dashboard/site -> "Windy Hub"; windy-calendar only -> "Windy Calender";
|
|
||||||
# windy-admin/telemetry -> "Windy Admin"). windy-pro here = its server/web side; the desktop
|
|
||||||
# app is Grant-owned and listed in its own section below.
|
|
||||||
OWNERS = {
|
|
||||||
"windy-chat": "Windy Chat", "windy-mail": "Windy Mail", "windy-calendar": "Windy Calender",
|
|
||||||
"Windy-Clone": "Windy Clone", "WindyCloud": "Windy Cloud", "windy-cloud-sites": "Windy Cloud",
|
|
||||||
"windy-cloud-domains": "Windy Cloud", "windy-cloud-vps": "Windy Cloud", "windy-search": "Windy Search",
|
|
||||||
"windy-connect": "Windy Connect", "windy-drops": "Windy Drops", "windy-registry": "Windy Drops",
|
|
||||||
"windy-code-web": "Windy Code", "windy-code": "Windy Code", "windy-traveler": "Windy Traveler",
|
|
||||||
"windytraveler-site": "Windy Traveler", "eternitas": "Eternitas", "windy-translate": "Windy Translate",
|
|
||||||
"windytranslate-site": "Windy Translate", "windy-hand": "Windy Hand", "windytalk": "Windy Talk",
|
|
||||||
"windy-pro": "Windy Hub", "windy-mind": "WIndy Mind", "windy-git": "Windy Git",
|
|
||||||
"windy-inbox": "Windy Drops"} # Windy Inbox build lead (09-24)
|
|
||||||
JOB = re.compile(r"^ ([A-Za-z0-9_-]+):\s*$")
|
|
||||||
|
|
||||||
|
|
||||||
def job_of(text: str, line: int) -> str | None:
|
|
||||||
"""The workflow job a line belongs to (2-space keys under `jobs:`)."""
|
|
||||||
in_jobs, job = False, None
|
|
||||||
for i, raw in enumerate(text.splitlines(), 1):
|
|
||||||
if raw.startswith("jobs:"):
|
|
||||||
in_jobs = True
|
|
||||||
elif in_jobs and JOB.match(raw):
|
|
||||||
job = JOB.match(raw).group(1)
|
|
||||||
elif raw and not raw[0].isspace() and not raw.startswith("jobs:"):
|
|
||||||
in_jobs = False
|
|
||||||
if i == line:
|
|
||||||
return job if in_jobs else None
|
|
||||||
return None
|
|
||||||
|
|
||||||
|
|
||||||
def grant_owned(repo: str, path: str, job: str | None, owned: list[dict]) -> bool:
|
|
||||||
for e in owned:
|
|
||||||
if e["repo"] != repo:
|
|
||||||
continue
|
|
||||||
if any(fnmatch.fnmatch(path, g) for g in e.get("paths") or []):
|
|
||||||
return True
|
|
||||||
if job and job in (e.get("jobs") or {}).get(path, []):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def split_grant(repo: str, sha: str, findings: list) -> tuple[list, list]:
|
|
||||||
"""(lane-owned, Grant-owned) findings at `sha`, by ci/grant-owned.yml, with
|
|
||||||
workflow lines attributed to their job exactly as the status page does."""
|
|
||||||
owned = (yaml.safe_load(OWNED.read_text()) or {}).get("grant_owned") or []
|
|
||||||
if not any(e["repo"] == repo for e in owned):
|
|
||||||
return list(findings), []
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
texts: dict[str, str] = {}
|
|
||||||
lane, grant = [], []
|
|
||||||
for f in findings:
|
|
||||||
job = None
|
|
||||||
if "/workflows/" in f.path:
|
|
||||||
if f.path not in texts:
|
|
||||||
texts[f.path] = cg._git(bare, "show", f"{sha}:{f.path}")
|
|
||||||
job = job_of(texts[f.path], f.line)
|
|
||||||
(grant if grant_owned(repo, f.path, job, owned) else lane).append(f)
|
|
||||||
return lane, grant
|
|
||||||
|
|
||||||
|
|
||||||
def scan(repo: str, owned: list[dict]):
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
if not bare.is_dir():
|
|
||||||
return None
|
|
||||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
|
||||||
sha = cg._git(bare, "rev-parse", head).strip()
|
|
||||||
out = {"sha": sha, "compute": [], "hygiene": [], "secrets": []}
|
|
||||||
texts: dict[str, str] = {}
|
|
||||||
for key, fs in (("compute", cg.check(repo, sha, head, True) or []),
|
|
||||||
("hygiene", hy.check(repo, sha, head, True) or []),
|
|
||||||
("secrets", sgd.check(repo, sha, head, True) or [])):
|
|
||||||
for f in fs:
|
|
||||||
job = None
|
|
||||||
if "/workflows/" in f.path:
|
|
||||||
if f.path not in texts:
|
|
||||||
texts[f.path] = cg._git(bare, "show", f"{sha}:{f.path}")
|
|
||||||
job = job_of(texts[f.path], f.line)
|
|
||||||
out[key].append((f, job, grant_owned(repo, f.path, job, owned)))
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def render(results: dict) -> str:
|
|
||||||
now = time.strftime("%Y-%m-%d %H:%MZ", time.gmtime())
|
|
||||||
lane = {k: 0 for k in ("compute", "hygiene", "secrets")}
|
|
||||||
grant = {k: 0 for k in ("compute", "hygiene", "secrets")}
|
|
||||||
for r in results.values():
|
|
||||||
for k in lane:
|
|
||||||
lane[k] += sum(1 for _, _, g in r.get(k, []) if not g)
|
|
||||||
grant[k] += sum(1 for _, _, g in r.get(k, []) if g)
|
|
||||||
L = [f"# Repo guards: live status (generated {now}; windy-git scripts/guards_report.py)",
|
|
||||||
"_Default branches only. WARN-only today; the orchestrator says \"block\" per guard when its LANE column is 0. "
|
|
||||||
"Grant-owned code (ci/grant-owned.yml) is listed separately and never holds up a block._", "",
|
|
||||||
"| Guard | Lane-owned findings | Grant-owned (proposals) | Ready to block? |", "|---|---|---|---|",
|
|
||||||
f"| compute-guard (Mind is the only door) | {lane['compute']} | {grant['compute']} | {'✅ YES' if lane['compute'] == 0 else '❌ not yet'} |",
|
|
||||||
f"| ci-hygiene (house rule 6) | {lane['hygiene']} | {grant['hygiene']} | {'✅ YES' if lane['hygiene'] == 0 else '❌ not yet'} |",
|
|
||||||
f"| secret-guard (no credentials in repos; hash only) | {lane['secrets']} | {grant['secrets']} | {'✅ YES' if lane['secrets'] == 0 else '❌ not yet'} |",
|
|
||||||
"", "## By repo (lane-owned)", "| Repo | owner | head | compute | hygiene | secrets | first items |", "|---|---|---|---|---|---|---|"]
|
|
||||||
for repo, r in sorted(results.items()):
|
|
||||||
c = [x for x in r["compute"] if not x[2]]
|
|
||||||
h = [x for x in r["hygiene"] if not x[2]]
|
|
||||||
s = [x for x in r.get("secrets", []) if not x[2]]
|
|
||||||
items = "; ".join(f"`{f.path}:{f.line}` {f.match}" for f, _, _ in (s + c + h)[:3]) or "clean ✅"
|
|
||||||
L.append(f"| {repo} | {OWNERS.get(repo, '?')} | {r['sha'][:7]} | {len(c)} | {len(h)} | {len(s)} | {items} |")
|
|
||||||
L += ["", "## Grant-owned (windy-pro desktop app + its build jobs): proposals only, not blocking"]
|
|
||||||
g = [(repo, f, job) for repo, r in sorted(results.items()) for k in ("compute", "hygiene", "secrets")
|
|
||||||
for f, job, own in r.get(k, []) if own]
|
|
||||||
L += [f"- {repo} `{f.path}:{f.line}`{f' (job {job})' if job else ''}: {f.match}" for repo, f, job in g] or ["- none"]
|
|
||||||
return "\n".join(L) + "\n"
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
owned = (yaml.safe_load(OWNED.read_text()) or {}).get("grant_owned") or []
|
|
||||||
results = {}
|
|
||||||
for repo in REPOS:
|
|
||||||
r = scan(repo, owned)
|
|
||||||
if r is not None:
|
|
||||||
results[repo] = r
|
|
||||||
sys.stdout.write(render(results))
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -53,8 +53,7 @@ REPOS = os.environ.get(
|
|||||||
"windy-chat windy-mail windy-calendar Windy-Clone WindyCloud windy-search windy-connect"
|
"windy-chat windy-mail windy-calendar Windy-Clone WindyCloud windy-search windy-connect"
|
||||||
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
" windy-drops windy-code-web windy-code windy-traveler windy-registry eternitas"
|
||||||
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
" windy-translate windytranslate-site windytraveler-site windy-hand"
|
||||||
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind"
|
" windy-cloud-sites windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-mind",
|
||||||
" windy-inbox windy-text windy-call windy-cell",
|
|
||||||
).split()
|
).split()
|
||||||
|
|
||||||
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
# Gitea run status -> GitHub status state. `skipped` is deliberately absent: a
|
||||||
@@ -75,23 +74,7 @@ MIRROR_TAG = "[GH#"
|
|||||||
# Posting them would put a permanent red X on every commit, and a signal that is
|
# Posting them would put a permanent red X on every commit, and a signal that is
|
||||||
# always red trains everyone to ignore red. Not posted until a rootless builder
|
# always red trains everyone to ignore red. Not posted until a rootless builder
|
||||||
# exists; that is a decision, recorded in docs/CUTOVER.md, not a failure.
|
# exists; that is a decision, recorded in docs/CUTOVER.md, not a failure.
|
||||||
# ...but NOT the no-Docker smoke jobs that replaced them (option A, 09-23):
|
NO_DAEMON_JOB = re.compile(r"docker", re.IGNORECASE)
|
||||||
# windy-search's "Boot smoke (no Docker)" matched plain `docker` and was hidden.
|
|
||||||
NO_DAEMON_JOB = re.compile(r"(?<!no )(?<!no-)(?<!without )docker", re.IGNORECASE)
|
|
||||||
# Image-build jobs whose NAME doesn't say docker (orchestrator 09-23, option A:
|
|
||||||
# each lane converts the job to a no-Docker smoke test; until then it is not
|
|
||||||
# posted). Format: "repo:workflow/job,...;repo2:...".
|
|
||||||
NO_DAEMON_NAMED: dict[str, set[str]] = {}
|
|
||||||
# eternitas:ci/build dropped 09-23: converted to a no-Docker ci/smoke (#179).
|
|
||||||
for _entry in os.environ.get("BRIDGE_NO_DAEMON", "").split(";"):
|
|
||||||
if ":" in _entry:
|
|
||||||
_repo, _jobs = _entry.split(":", 1)
|
|
||||||
NO_DAEMON_NAMED[_repo.strip()] = {j.strip() for j in _jobs.split(",") if j.strip()}
|
|
||||||
|
|
||||||
|
|
||||||
def needs_daemon(repo: str, wf: str, job: str) -> bool:
|
|
||||||
"""True for image-build jobs, which cannot run here (no Docker daemon, I-5)."""
|
|
||||||
return bool(NO_DAEMON_JOB.search(job)) or f"{wf}/{job}" in NO_DAEMON_NAMED.get(repo, ())
|
|
||||||
|
|
||||||
# Jobs Grant ruled NON-BLOCKING (GRANT_DECISIONS_2026-09-23): still run on
|
# Jobs Grant ruled NON-BLOCKING (GRANT_DECISIONS_2026-09-23): still run on
|
||||||
# Windy Git and visible there, but not posted to GitHub, so they cannot turn a
|
# Windy Git and visible there, but not posted to GitHub, so they cannot turn a
|
||||||
@@ -219,17 +202,7 @@ def sync_prs(repo: str) -> list[str]:
|
|||||||
wanted.add(tag)
|
wanted.add(tag)
|
||||||
heads.append(pr["head"]["sha"])
|
heads.append(pr["head"]["sha"])
|
||||||
if tag in ours:
|
if tag in ours:
|
||||||
cur = (ours[tag].get("base") or {}).get("ref")
|
continue
|
||||||
if cur is None or cur == pr["base"]["ref"]: # unknown base: never guess, leave it
|
|
||||||
continue
|
|
||||||
# Retargeted on GitHub (e.g. a stacked PR moved to main after its
|
|
||||||
# parent merged). The mirror kept the OLD base, so workflows filtered
|
|
||||||
# on the base (`pull_request: branches: [main]`) silently stopped
|
|
||||||
# running: eternitas #167, 09-23. Replace the mirror: an "edited"
|
|
||||||
# event triggers nothing, a freshly opened PR runs CI at once.
|
|
||||||
gitea("PATCH", f"/repos/{WG_OWNER}/{repo}/pulls/{ours[tag]['number']}", {"state": "closed"})
|
|
||||||
print(f" {repo}: GH#{pr['number']} retargeted {cur} -> "
|
|
||||||
f"{pr['base']['ref']}: replacing its mirror PR")
|
|
||||||
st, _ = gitea(
|
st, _ = gitea(
|
||||||
"POST",
|
"POST",
|
||||||
f"/repos/{WG_OWNER}/{repo}/pulls",
|
f"/repos/{WG_OWNER}/{repo}/pulls",
|
||||||
@@ -311,7 +284,7 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
break
|
break
|
||||||
latest: dict[str, dict] = {}
|
latest: dict[str, dict] = {}
|
||||||
for r in runs:
|
for r in runs:
|
||||||
if r["head_sha"] != sha or needs_daemon(repo, r["workflow_id"].removesuffix(".yml"), r["name"]):
|
if r["head_sha"] != sha or NO_DAEMON_JOB.search(r["name"]):
|
||||||
continue
|
continue
|
||||||
if f"{r['workflow_id'].removesuffix('.yml')}/{r['name']}" in NON_BLOCKING.get(repo, ()):
|
if f"{r['workflow_id'].removesuffix('.yml')}/{r['name']}" in NON_BLOCKING.get(repo, ()):
|
||||||
continue
|
continue
|
||||||
@@ -321,9 +294,9 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
# Queued jobs: `pending` where nothing newer has been picked up. A re-run
|
# Queued jobs: `pending` where nothing newer has been picked up. A re-run
|
||||||
# queued behind an old failure must read pending, not the stale red.
|
# queued behind an old failure must read pending, not the stale red.
|
||||||
for q in queued_jobs(repo, sha):
|
for q in queued_jobs(repo, sha):
|
||||||
wf = q["workflow_id"].removesuffix(".yml")
|
if NO_DAEMON_JOB.search(q["name"]):
|
||||||
if needs_daemon(repo, wf, q["name"]):
|
|
||||||
continue
|
continue
|
||||||
|
wf = q["workflow_id"].removesuffix(".yml")
|
||||||
if f"{wf}/{q['name']}" in NON_BLOCKING.get(repo, ()):
|
if f"{wf}/{q['name']}" in NON_BLOCKING.get(repo, ()):
|
||||||
continue
|
continue
|
||||||
ctx = f"windy-git/{wf}/{q['name']}"
|
ctx = f"windy-git/{wf}/{q['name']}"
|
||||||
@@ -371,58 +344,34 @@ def post_statuses(repo: str, sha: str) -> None:
|
|||||||
|
|
||||||
|
|
||||||
GUARD_CTX = "windy-git/compute-guard"
|
GUARD_CTX = "windy-git/compute-guard"
|
||||||
HYGIENE_CTX = "windy-git/ci-hygiene"
|
|
||||||
SECRET_CTX = "windy-git/secret-guard"
|
|
||||||
|
|
||||||
|
|
||||||
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
def post_compute_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
||||||
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only)."""
|
"""Windy Mind is the only door to AI compute: flag direct provider use (warn-only).
|
||||||
_post_guard("compute_guard", GUARD_CTX, repo, sha, default_branch, is_default_head)
|
|
||||||
|
|
||||||
|
Non-fatal and never a fake OK: if the guard can't run, nothing is posted.
|
||||||
def post_secret_guard(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
"""
|
||||||
"""No live credential in a bridged repo (leak hunt 09-24). Findings carry sha256[:8] only."""
|
|
||||||
_post_guard("secret_guard", SECRET_CTX, repo, sha, default_branch, is_default_head)
|
|
||||||
|
|
||||||
|
|
||||||
def post_ci_hygiene(repo: str, sha: str, default_branch: str, is_default_head: bool) -> None:
|
|
||||||
"""House rule 6: lockfile-only installs, pinned images, no host-port services (warn-only)."""
|
|
||||||
_post_guard("ci_hygiene", HYGIENE_CTX, repo, sha, default_branch, is_default_head)
|
|
||||||
|
|
||||||
|
|
||||||
def _post_guard(modname: str, ctx: str, repo: str, sha: str, default_branch: str,
|
|
||||||
is_default_head: bool) -> None:
|
|
||||||
"""One code path for every repo-scanning guard. Non-fatal and never a fake OK:
|
|
||||||
if the guard can't run, nothing is posted."""
|
|
||||||
try:
|
try:
|
||||||
import importlib
|
import compute_guard as cg # same directory; loaded lazily so the bridge never depends on it
|
||||||
|
|
||||||
g = importlib.import_module(modname) # same directory; lazy so the bridge never depends on it
|
findings = cg.check(repo, sha, default_branch, is_default_head)
|
||||||
findings = g.check(repo, sha, default_branch, is_default_head)
|
except Exception as e: # noqa: BLE001 — the guard must never break CI signals
|
||||||
except Exception as e: # noqa: BLE001 — a guard must never break CI signals
|
print(f" {repo}@{sha[:7]} compute-guard skipped ({type(e).__name__}: {str(e)[:80]})")
|
||||||
print(f" {repo}@{sha[:7]} {ctx} skipped ({type(e).__name__}: {str(e)[:80]})")
|
|
||||||
return
|
return
|
||||||
if findings is None:
|
if findings is None:
|
||||||
return
|
return
|
||||||
try: # Grant-owned code never blocks (orchestrator 09-23); lazy like the guards
|
state, desc, first = cg.status_for(findings, whole_tree=is_default_head)
|
||||||
import importlib
|
|
||||||
|
|
||||||
lane, grant = importlib.import_module("guards_report").split_grant(repo, sha, findings)
|
|
||||||
except Exception as e: # noqa: BLE001 — can't tell whose code: warn, never block
|
|
||||||
print(f" {repo}@{sha[:7]} {ctx}: Grant-owned split failed ({type(e).__name__}); WARN only")
|
|
||||||
lane, grant = [], list(findings)
|
|
||||||
state, desc, first = g.status_for(lane, whole_tree=is_default_head, grant=grant)
|
|
||||||
st, existing = github("GET", f"/repos/{GH_OWNER}/{repo}/commits/{sha}/statuses?per_page=100")
|
st, existing = github("GET", f"/repos/{GH_OWNER}/{repo}/commits/{sha}/statuses?per_page=100")
|
||||||
for s in existing or []: # newest first: compare the latest guard status only
|
for s in existing or []: # newest first: compare the latest guard status only
|
||||||
if s["context"] == ctx:
|
if s["context"] == GUARD_CTX:
|
||||||
if (s["state"], s.get("description")) == (state, desc):
|
if (s["state"], s.get("description")) == (state, desc):
|
||||||
return
|
return
|
||||||
break
|
break
|
||||||
url = (f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}/{first.path}#L{first.line}"
|
url = (f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}/{first.path}#L{first.line}"
|
||||||
if first else f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}")
|
if first else f"{PUBLIC}/{WG_OWNER}/{repo}/src/commit/{sha}")
|
||||||
st, _ = github("POST", f"/repos/{GH_OWNER}/{repo}/statuses/{sha}",
|
st, _ = github("POST", f"/repos/{GH_OWNER}/{repo}/statuses/{sha}",
|
||||||
{"state": state, "context": ctx, "description": desc, "target_url": url})
|
{"state": state, "context": GUARD_CTX, "description": desc, "target_url": url})
|
||||||
print(f" {repo}@{sha[:7]} {ctx} = {state} ({len(findings)} finding(s)) -> {st}")
|
print(f" {repo}@{sha[:7]} {GUARD_CTX} = {state} ({len(findings)} finding(s)) -> {st}")
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
@@ -443,8 +392,6 @@ def main() -> int:
|
|||||||
for sha in dict.fromkeys(shas):
|
for sha in dict.fromkeys(shas):
|
||||||
post_statuses(repo, sha)
|
post_statuses(repo, sha)
|
||||||
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
post_compute_guard(repo, sha, default_branch, sha == default_head)
|
||||||
post_ci_hygiene(repo, sha, default_branch, sha == default_head)
|
|
||||||
post_secret_guard(repo, sha, default_branch, sha == default_head)
|
|
||||||
except Exception as e: # one repo's failure must not hide the others'
|
except Exception as e: # one repo's failure must not hide the others'
|
||||||
print(f" FAILED {repo}: {e}")
|
print(f" FAILED {repo}: {e}")
|
||||||
failed = 1
|
failed = 1
|
||||||
|
|||||||
@@ -1,118 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Weekly: every PUBLIC repo in Grant's GitHub accounts, full history (every object,
|
|
||||||
reachable or not, incl. PR refs), for secret-shaped strings. Leak hunt 09-24: a
|
|
||||||
real bot token sat in a public test fixture for five months.
|
|
||||||
|
|
||||||
Output is hash + location only, never a value (house rule 10). Known fakes are
|
|
||||||
excused by ci/secret-guard-allow.yml (same file as secret-guard; the repo NAME is
|
|
||||||
matched across accounts). Runs on Veron as user1-gpu (gh is logged in there):
|
|
||||||
|
|
||||||
python3 scripts/public_secret_scan.py [--out FILE] [--owners a,b,...]
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import fnmatch
|
|
||||||
import json
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
||||||
import secret_shapes as ss # noqa: E402
|
|
||||||
|
|
||||||
OWNERS = ["sneakyfree", "VERONTECH", "Windstorm-Institute", "Windstorm-Labs", "Public-Streamer"]
|
|
||||||
WORK = Path.home() / "leakscan" / "public"
|
|
||||||
MAX_BLOB = 20_000_000
|
|
||||||
|
|
||||||
|
|
||||||
def _run(*a: str) -> subprocess.CompletedProcess:
|
|
||||||
return subprocess.run(a, capture_output=True)
|
|
||||||
|
|
||||||
|
|
||||||
def blob_hits(bare: Path) -> dict[str, list[tuple[str, str]]]:
|
|
||||||
"""{blob: [(kind, hash8)]} over every blob object in the repo."""
|
|
||||||
chk = _run("git", "-C", str(bare), "cat-file", "--batch-all-objects",
|
|
||||||
"--batch-check=%(objectname) %(objecttype) %(objectsize)")
|
|
||||||
blobs = [p[0] for p in (ln.split() for ln in chk.stdout.decode().splitlines())
|
|
||||||
if len(p) == 3 and p[1] == "blob" and int(p[2]) < MAX_BLOB]
|
|
||||||
if not blobs:
|
|
||||||
return {}
|
|
||||||
import threading
|
|
||||||
p = subprocess.Popen(["git", "-C", str(bare), "cat-file", "--batch"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
def feed() -> None: # separate thread: writing everything first deadlocks (both pipes fill)
|
|
||||||
p.stdin.write(("\n".join(blobs) + "\n").encode())
|
|
||||||
p.stdin.close()
|
|
||||||
threading.Thread(target=feed, daemon=True).start()
|
|
||||||
out: dict[str, list[tuple[str, str]]] = {}
|
|
||||||
for _ in blobs:
|
|
||||||
hdr = p.stdout.readline().split()
|
|
||||||
data = p.stdout.read(int(hdr[2]))
|
|
||||||
p.stdout.read(1)
|
|
||||||
found = ss.find(data.decode("utf-8", "ignore"))
|
|
||||||
if found:
|
|
||||||
out[hdr[0].decode()] = found
|
|
||||||
p.wait()
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def locate(bare: Path, blob: str) -> dict:
|
|
||||||
lg = _run("git", "-C", str(bare), "log", "--all", "--format=@@%H %cI", "--name-only",
|
|
||||||
f"--find-object={blob}").stdout.decode()
|
|
||||||
commits, paths = [], set()
|
|
||||||
for line in lg.splitlines():
|
|
||||||
if line.startswith("@@"):
|
|
||||||
commits.append(line[2:].split())
|
|
||||||
elif line.strip():
|
|
||||||
paths.add(line.strip())
|
|
||||||
head = _run("git", "-C", str(bare), "ls-tree", "-r", "HEAD").stdout.decode()
|
|
||||||
first = commits[-1] if commits else ["unreachable", "-"]
|
|
||||||
return {"paths": sorted(paths), "first_commit": first[0][:10], "first_date": first[1],
|
|
||||||
"in_head": blob in head}
|
|
||||||
|
|
||||||
|
|
||||||
def excused(repo: str, kind: str, h: str, paths: list[str], allow: dict) -> bool:
|
|
||||||
a = allow.get(repo) or {"hashes": set(), "paths": []}
|
|
||||||
if kind in {"private key block"}:
|
|
||||||
return bool(paths) and all(any(kind in k and fnmatch.fnmatch(p, g) for g, k in a["paths"]) for p in paths)
|
|
||||||
return h in a["hashes"]
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
|
||||||
ap = argparse.ArgumentParser()
|
|
||||||
ap.add_argument("--out", default=str(WORK / "latest.json"))
|
|
||||||
ap.add_argument("--owners", default=",".join(OWNERS))
|
|
||||||
args = ap.parse_args()
|
|
||||||
import secret_guard as sg
|
|
||||||
allow = sg.load_allow()
|
|
||||||
WORK.mkdir(parents=True, exist_ok=True)
|
|
||||||
rows, scanned, errors = [], [], []
|
|
||||||
for owner in args.owners.split(","):
|
|
||||||
lst = _run("gh", "repo", "list", owner, "--limit", "1000", "--visibility", "public", "--json", "name")
|
|
||||||
for r in json.loads(lst.stdout or b"[]"):
|
|
||||||
name = r["name"]
|
|
||||||
bare = WORK / owner / f"{name}.git"
|
|
||||||
if bare.exists():
|
|
||||||
c = _run("git", "-C", str(bare), "remote", "update", "--prune")
|
|
||||||
else:
|
|
||||||
bare.parent.mkdir(parents=True, exist_ok=True)
|
|
||||||
c = _run("gh", "repo", "clone", f"{owner}/{name}", str(bare), "--", "--mirror", "-q")
|
|
||||||
if c.returncode:
|
|
||||||
errors.append(f"{owner}/{name}")
|
|
||||||
continue
|
|
||||||
scanned.append(f"{owner}/{name}")
|
|
||||||
for blob, found in blob_hits(bare).items():
|
|
||||||
loc = locate(bare, blob)
|
|
||||||
for kind, h in sorted(set(found)):
|
|
||||||
rows.append({"repo": f"{owner}/{name}", "kind": kind, "hash8": h, **loc,
|
|
||||||
"excused": excused(name, kind, h, loc["paths"], allow)})
|
|
||||||
Path(args.out).write_text(json.dumps({"scanned": scanned, "errors": errors, "rows": rows}, indent=1))
|
|
||||||
new = [r for r in rows if not r["excused"]]
|
|
||||||
print(f"scanned {len(scanned)} public repos, {len(errors)} errors, {len(rows)} secret-shaped, {len(new)} NOT excused")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
sys.exit(main())
|
|
||||||
@@ -16,10 +16,7 @@
|
|||||||
# itself, so Windy Git is never left behind GitHub.
|
# itself, so Windy Git is never left behind GitHub.
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
repo="${1:?repo}"; branch="${2:?branch}"; want="${3:?sha prefix}"
|
repo="${1:?repo}"; branch="${2:?branch}"; want="${3:?sha prefix}"
|
||||||
# wg-q lives in the INVOKING user's ~/bin; under sudo, ~ is /root.
|
G="sudo docker exec -u git windy-git-gitea-1 git -C /data/git/repositories/windyadmin/${repo}.git"
|
||||||
WGQ="${WGQ:-$(getent passwd "${SUDO_USER:-$USER}" | cut -d: -f6)/bin/wg-q}"
|
|
||||||
# Gitea stores repositories LOWERCASED on disk (WindyCloud -> windycloud.git).
|
|
||||||
G="sudo docker exec -u git windy-git-gitea-1 git -C /data/git/repositories/windyadmin/${repo,,}.git"
|
|
||||||
|
|
||||||
head=$($G rev-parse "refs/heads/${branch}")
|
head=$($G rev-parse "refs/heads/${branch}")
|
||||||
[[ "$head" == "$want"* ]] || { echo "refusing: ${branch} is at ${head:0:7}, not ${want}"; exit 1; }
|
[[ "$head" == "$want"* ]] || { echo "refusing: ${branch} is at ${head:0:7}, not ${want}"; exit 1; }
|
||||||
@@ -45,7 +42,7 @@ until [ "$(systemctl show windygit-sync -p ExecMainStartTimestampMonotonic --val
|
|||||||
done
|
done
|
||||||
echo "restored by sync: ${branch} = ${head:0:7}"
|
echo "restored by sync: ${branch} = ${head:0:7}"
|
||||||
sleep 5
|
sleep 5
|
||||||
"$WGQ" <<SQL
|
~/bin/wg-q <<SQL
|
||||||
select ar.index, ar.workflow_id, ar.event, ar.status, to_char(to_timestamp(ar.created),'HH24:MI:SS')
|
select ar.index, ar.workflow_id, ar.event, ar.status, to_char(to_timestamp(ar.created),'HH24:MI:SS')
|
||||||
from action_run ar join repository r on r.id = ar.repo_id
|
from action_run ar join repository r on r.id = ar.repo_id
|
||||||
where r.name = '${repo}' and ar.commit_sha = '${head}' order by ar.id desc limit 6;
|
where r.name = '${repo}' and ar.commit_sha = '${head}' order by ar.id desc limit 6;
|
||||||
|
|||||||
@@ -1,131 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Secret guard: no live credential lands in a bridged repo (leak hunt 09-24).
|
|
||||||
|
|
||||||
Same walker, cache and GitHub posting as compute_guard / ci_hygiene
|
|
||||||
(`windy-git/secret-guard`), but over EVERY text file, and a finding carries only
|
|
||||||
"<kind> #<sha256[:8]>", never the value (house rule 10). Known fakes are allowed
|
|
||||||
BY HASH in ci/secret-guard-allow.yml (repo + hashes + reason).
|
|
||||||
|
|
||||||
sudo python3 scripts/secret_guard.py report [repo ...]
|
|
||||||
"""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import fnmatch
|
|
||||||
import hashlib
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
import sys
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
||||||
import compute_guard as cg # noqa: E402 (shared walker, cache)
|
|
||||||
import secret_shapes as ss # noqa: E402
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
|
||||||
ALLOW_FILE = Path(os.environ.get("SECRET_GUARD_ALLOW", ROOT / "ci" / "secret-guard-allow.yml"))
|
|
||||||
MODE = os.environ.get("SECRET_GUARD_MODE", "warn")
|
|
||||||
# Kinds that only WARN (rolled out warn-first); empty = every kind blocks in block mode.
|
|
||||||
WARN_KINDS = {k for k in os.environ.get("SECRET_GUARD_WARN_KINDS", "").split(",") if k}
|
|
||||||
NEVER = re.compile(r"(^|/)(node_modules|vendor|third_party)/")
|
|
||||||
|
|
||||||
|
|
||||||
def path_ok(path: str) -> bool:
|
|
||||||
return not NEVER.search(path)
|
|
||||||
|
|
||||||
|
|
||||||
# A private-key match is only its BEGIN line, so its hash is the same everywhere:
|
|
||||||
# those are allowed by PATH (entries with `paths` + `kinds`), everything else by HASH.
|
|
||||||
PATH_ONLY_KINDS = {"private key block"}
|
|
||||||
|
|
||||||
|
|
||||||
def load_allow(path: Path = ALLOW_FILE) -> dict[str, dict]:
|
|
||||||
"""{repo: {"hashes": {hash8}, "paths": [(glob, {kind})]}}; every entry needs a reason."""
|
|
||||||
data = yaml.safe_load(path.read_text()) if path.exists() else {}
|
|
||||||
out: dict[str, dict] = {}
|
|
||||||
for e in (data or {}).get("allow") or []:
|
|
||||||
if not (e.get("repo") and (e.get("hashes") or (e.get("paths") and e.get("kinds")))
|
|
||||||
and str(e.get("reason", "")).strip()):
|
|
||||||
raise ValueError(f"allow entry needs repo, hashes (or paths + kinds) and a reason: {e}")
|
|
||||||
if e.get("paths") and not set(e["kinds"]) <= PATH_ONLY_KINDS:
|
|
||||||
raise ValueError(f"path allows are only for {sorted(PATH_ONLY_KINDS)}: {e}")
|
|
||||||
r = out.setdefault(e["repo"], {"hashes": set(), "paths": []})
|
|
||||||
r["hashes"].update(str(h) for h in e.get("hashes") or [])
|
|
||||||
r["paths"] += [(g, set(e["kinds"])) for g in e.get("paths") or []]
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
def scan_line(path: str, text: str) -> list[tuple[str, str]]:
|
|
||||||
return [(kind, f"{kind} #{h}") for kind, h in ss.find(text)]
|
|
||||||
|
|
||||||
|
|
||||||
def _drop_allowed(repo: str, findings, allow: dict[str, dict]):
|
|
||||||
a = allow.get(repo) or {"hashes": set(), "paths": []}
|
|
||||||
|
|
||||||
def ok(f) -> bool:
|
|
||||||
if f.kind in PATH_ONLY_KINDS:
|
|
||||||
return any(f.kind in kinds and fnmatch.fnmatch(f.path, g) for g, kinds in a["paths"])
|
|
||||||
return f.match.rsplit("#", 1)[-1] in a["hashes"]
|
|
||||||
return [f for f in findings if not ok(f)]
|
|
||||||
|
|
||||||
|
|
||||||
def check(repo: str, sha: str, default_branch: str, is_default_head: bool):
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
if not bare.is_dir() or not cg.fetched(bare, sha): # pushed after the fetch: next cycle
|
|
||||||
return None
|
|
||||||
allow = load_allow()
|
|
||||||
rules = hashlib.sha256(("|".join(rx.pattern for _, rx in ss.PATTERNS) + ss.PREFILTER).encode()).hexdigest()[:8]
|
|
||||||
kw = dict(line_fn=scan_line, path_ok=path_ok)
|
|
||||||
if is_default_head:
|
|
||||||
fs = cg.cached_scan(f"sec-tree:{repo}:{sha}:{rules}",
|
|
||||||
lambda: cg.scan_tree(repo, bare, sha, [], prefilter=ss.PREFILTER, **kw))
|
|
||||||
else:
|
|
||||||
fs = cg.cached_scan(f"sec-pr:{repo}:{sha}:{rules}",
|
|
||||||
lambda: cg.scan_added(repo, bare, f"refs/heads/{default_branch}", sha, [], **kw))
|
|
||||||
return _drop_allowed(repo, fs, allow)
|
|
||||||
|
|
||||||
|
|
||||||
def status_for(findings, whole_tree: bool, grant=()):
|
|
||||||
"""Same contract as the other guards. `grant` findings never block."""
|
|
||||||
scope = "in tree" if whole_tree else "added"
|
|
||||||
if not findings and grant:
|
|
||||||
g, n = grant[0], len(grant)
|
|
||||||
return "success", f"⚠ WARN (Grant-owned, not blocking): {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {g.path}:{g.line} {g.match}"[:140], g
|
|
||||||
if not findings:
|
|
||||||
return "success", f"OK: no secret-shaped strings {scope}", None
|
|
||||||
f, n = findings[0], len(findings)
|
|
||||||
soft = MODE != "block" or all(x.kind in WARN_KINDS for x in findings)
|
|
||||||
state = "success" if soft else "failure"
|
|
||||||
lead = "⚠ WARN (not blocking)" if soft else "BLOCKED"
|
|
||||||
if not soft:
|
|
||||||
f = next(x for x in findings if x.kind not in WARN_KINDS)
|
|
||||||
return state, f"{lead}: {n} secret-shaped string{'s' if n > 1 else ''} {scope}, e.g. {f.path}:{f.line} {f.match}"[:140], f
|
|
||||||
|
|
||||||
|
|
||||||
def report(repos: list[str]) -> int:
|
|
||||||
allow = load_allow()
|
|
||||||
total = 0
|
|
||||||
for repo in repos:
|
|
||||||
bare = cg.WORK / f"{repo}.git"
|
|
||||||
if not bare.is_dir():
|
|
||||||
continue
|
|
||||||
head = cg._git(bare, "symbolic-ref", "--short", "HEAD").strip()
|
|
||||||
sha = cg._git(bare, "rev-parse", head).strip()
|
|
||||||
fs = _drop_allowed(repo, cg.scan_tree(repo, bare, sha, [], line_fn=scan_line, path_ok=path_ok,
|
|
||||||
prefilter=ss.PREFILTER), allow)
|
|
||||||
total += len(fs)
|
|
||||||
print(f"## {repo} ({head} {sha[:7]}): {len(fs)} finding(s)")
|
|
||||||
for f in fs:
|
|
||||||
print(f" {f.path}:{f.line} {f.match}")
|
|
||||||
print(f"TOTAL {total}")
|
|
||||||
return 0
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
if len(sys.argv) >= 2 and sys.argv[1] == "report":
|
|
||||||
default = os.environ.get("BRIDGE_REPOS", "").split() or sorted(
|
|
||||||
p.name.removesuffix(".git") for p in cg.WORK.glob("*.git"))
|
|
||||||
sys.exit(report(sys.argv[2:] or default))
|
|
||||||
sys.exit(__doc__)
|
|
||||||
@@ -1,45 +0,0 @@
|
|||||||
"""Secret-shaped strings, shared by secret_guard (bridged repos) and
|
|
||||||
public_secret_scan (weekly, every public repo). A finding NEVER carries the value:
|
|
||||||
only its kind and sha256[:8] (house rule 10). Leak hunt 09-24: @Windy_0_bot's
|
|
||||||
token sat in a public repo's test fixture for five months."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import hashlib
|
|
||||||
import re
|
|
||||||
|
|
||||||
# (kind, regex). Order matters only for readability; each match is reported once.
|
|
||||||
PATTERNS: list[tuple[str, re.Pattern[str]]] = [
|
|
||||||
("telegram bot token", re.compile(r"(?<![0-9])[0-9]{8,10}:[A-Za-z0-9_-]{35}(?![A-Za-z0-9_-])")),
|
|
||||||
("github token", re.compile(r"\b(?:gh[pousr]_[A-Za-z0-9]{36}|github_pat_[A-Za-z0-9_]{82})\b")),
|
|
||||||
("aws access key", re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b")),
|
|
||||||
("slack token", re.compile(r"\bxox[abprs]-[A-Za-z0-9-]{10,}")),
|
|
||||||
("anthropic key", re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}")),
|
|
||||||
("openai key", re.compile(r"\bsk-(?:proj-|svcacct-)?(?!ant-)[A-Za-z0-9_-]{32,}")),
|
|
||||||
("stripe live key", re.compile(r"\b[rs]k_live_[A-Za-z0-9]{20,}")),
|
|
||||||
("google api key", re.compile(r"\bAIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])")),
|
|
||||||
# Twilio (Windy Text 10-01: a live auth token sat in test files for months). An auth token
|
|
||||||
# is a bare 32-hex with no prefix, so it is only caught when ASSIGNED to a secret-ish name.
|
|
||||||
("twilio sid/api key", re.compile(r"\b(?:AC|SK)[0-9a-f]{32}\b")),
|
|
||||||
("32-hex secret assignment", re.compile(
|
|
||||||
r"(?i)\b[a-z0-9_.-]*(?:token|secret|key|password)[a-z0-9_.-]*[\"']?\s*[:=]\s*[\"']?(?P<v>(?<![0-9a-f])[0-9a-f]{32}(?![0-9a-f]))")),
|
|
||||||
("private key block", re.compile(r"-----BEGIN (?:RSA |EC |DSA |OPENSSH |ENCRYPTED )?PRIVATE KEY-----")),
|
|
||||||
]
|
|
||||||
|
|
||||||
# git grep -E (POSIX ERE) prefilter: cheap superset of PATTERNS.
|
|
||||||
PREFILTER = ("[0-9]{8,10}:[A-Za-z0-9_-]{35}|gh[pousr]_[A-Za-z0-9]{36}|github_pat_|(AKIA|ASIA)[0-9A-Z]{16}"
|
|
||||||
"|xox[abprs]-|sk-ant-|sk-[A-Za-z0-9_-]{32}|sk-proj-|[rs]k_live_|AIza[0-9A-Za-z_-]{35}"
|
|
||||||
"|-----BEGIN [A-Z ]*PRIVATE KEY-----|(AC|SK)[0-9a-f]{32}|[0-9a-fA-F]{32}")
|
|
||||||
|
|
||||||
|
|
||||||
def h8(value: str | bytes) -> str:
|
|
||||||
return hashlib.sha256(value.encode() if isinstance(value, str) else value).hexdigest()[:8]
|
|
||||||
|
|
||||||
|
|
||||||
def find(text: str) -> list[tuple[str, str]]:
|
|
||||||
"""[(kind, hash8)] for every secret-shaped string in `text`. Values never leave."""
|
|
||||||
out = []
|
|
||||||
for kind, rx in PATTERNS:
|
|
||||||
for m in rx.finditer(text):
|
|
||||||
out.append((kind, h8(m.group('v') if 'v' in rx.groupindex else m.group(0))))
|
|
||||||
return out
|
|
||||||
@@ -38,7 +38,7 @@ FAILED=0
|
|||||||
|
|
||||||
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
# Repos Windy Git tracks FROM GitHub. Remove a repo from this list at the moment
|
||||||
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
# it flips to Windy-Git-first, or the sync will fight its authors and win.
|
||||||
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro windy-inbox windy-text windy-call windy-cell}"
|
REPOS="${SYNC_REPOS:-windy-calendar windy-search windy-registry Windy-Clone WindyCloud windy-cloud-sites windy-mind eternitas windy-agent windy-git windy-chat windy-mail windy-connect windy-drops windy-code-web windy-code windy-traveler windy-translate windytranslate-site windytraveler-site windy-hand windy-cloud-domains windy-cloud-vps windytalk windy-pro}"
|
||||||
|
|
||||||
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
# Repos whose TAGS must not reach Windy Git. A tag push fires `on: push: tags`
|
||||||
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
# workflows; windy-pro's build-electron is a matrix over ubuntu/macos/windows-
|
||||||
|
|||||||
Reference in New Issue
Block a user