From 9a7030351bf0e9a1ccf68cb19a4b3b3a3cd71b49 Mon Sep 17 00:00:00 2001 From: Grant Whitmer Date: Wed, 12 Aug 2026 13:41:37 -0400 Subject: [PATCH] =?UTF-8?q?G7.6:=20fleet=20canary=20=E2=80=94=20probes=20w?= =?UTF-8?q?hat=20a=20user=20does,=20not=20what=20is=20cheap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Today's outage is the whole design brief: /health returned 200 for the entire hour that login was dead. A canary watching /health would have stayed green while nobody in the ecosystem could sign in. So the login probe is here and it is the one that matters. Three rules it obeys: - never green for something it did not prove (I-8) - alert on TRANSITIONS, not every run — a canary people filter is a dead canary, which is how the last one sat 37 days dead unnoticed - run where the watched thing cannot take it down: Veron 1, never Kit 0 Two independent signals, so losing one still leaves the other: an email via Resend on state change, and a non-zero exit that turns the CI run red in the forge itself. Alerts say what broke in human terms — 'a human can actually sign in' — rather than only naming an endpoint. Verified against production: 7/7 green including login at 17.1s; a forced 404 reports DOWN; a 1s threshold reports SLOW at 23.3s. Co-Authored-By: Claude Opus 5 --- .gitea/workflows/canary.yml | 48 +++++++ api/tests/test_invariants.py | 33 +++++ scripts/canary.py | 237 +++++++++++++++++++++++++++++++++++ 3 files changed, 318 insertions(+) create mode 100644 .gitea/workflows/canary.yml create mode 100755 scripts/canary.py diff --git a/.gitea/workflows/canary.yml b/.gitea/workflows/canary.yml new file mode 100644 index 0000000..9fb0ba7 --- /dev/null +++ b/.gitea/workflows/canary.yml @@ -0,0 +1,48 @@ +# The fleet canary (G7.6). +# +# Runs on Veron 1, DELIBERATELY not on Kit 0. A canary hosted on the box it +# watches dies with that box and reports nothing at the exact moment it matters. +# +# The previous fleet canary (kit-army-config/docs/deployed-state.json) sat dead +# for 37+ days because its workflow returned startup_failure on every run and +# nothing watched the watcher. This one has two independent signals: an email on +# state change, and a red CI run in the forge. Losing one still leaves the other. + +name: canary + +on: + schedule: + # Every 10 minutes. Frequent enough that an outage is measured in minutes, + # infrequent enough that the login probe (~18s of real work on a loaded box) + # is not itself a load source. + - cron: "*/10 * * * *" + workflow_dispatch: + +jobs: + probe: + runs-on: veron-1 + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # State persists between runs so "still broken" can be told apart from + # "just broke" — that is what keeps this from emailing every 10 minutes + # during an outage, and a canary people filter is a dead canary. + - name: restore canary state + uses: actions/cache@v4 + with: + path: canary-state.json + key: canary-state-${{ github.run_id }} + restore-keys: canary-state- + + - name: probe + env: + RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }} + CANARY_LOGIN_EMAIL: ${{ secrets.CANARY_LOGIN_EMAIL }} + CANARY_LOGIN_PASSWORD: ${{ secrets.CANARY_LOGIN_PASSWORD }} + CANARY_ALERT_TO: ${{ secrets.CANARY_ALERT_TO }} + run: python3 scripts/canary.py diff --git a/api/tests/test_invariants.py b/api/tests/test_invariants.py index 1877cd4..2328f49 100644 --- a/api/tests/test_invariants.py +++ b/api/tests/test_invariants.py @@ -504,3 +504,36 @@ def test_g73_every_job_has_a_timeout(): for half an hour.""" for wf in ROOT.rglob(".gitea/workflows/*.y*ml"): assert "timeout-minutes:" in wf.read_text(), f"{wf.name}: no job timeout" + + +# -------------------------------------------------------------------------- +# G7.6 — the canary must watch what users do, and must not live on Kit 0 +# -------------------------------------------------------------------------- +def test_g76_canary_probes_login_not_just_health(): + """/health returned 200 for the entire 2026-08-12 outage while login was + dead. A canary that only watches health is decorative.""" + src = (ROOT / "scripts" / "canary.py").read_text() + assert "identity.login" in src + assert "/api/v1/auth/login" in src + + +def test_g76_canary_does_not_run_on_kit_zero(): + """A canary hosted on the box it watches dies with that box, and reports + nothing at the exact moment it matters.""" + wf = (ROOT / ".gitea" / "workflows" / "canary.yml").read_text() + assert "runs-on: veron-1" in wf + assert "72.60.118.54" not in wf + + +def test_g76_canary_alerts_on_transition_not_every_run(): + """A canary that emails every 10 minutes gets filtered, and a filtered + canary is a dead canary.""" + src = (ROOT / "scripts" / "canary.py").read_text() + assert "newly_bad" in src and "recovered" in src + + +def test_g76_canary_has_two_independent_signals(): + """Email AND a red CI run. The last fleet canary died silently because it + had one signal and nothing watched the watcher.""" + src = (ROOT / "scripts" / "canary.py").read_text() + assert "return 1 if any" in src, "canary must exit non-zero so CI goes red" diff --git a/scripts/canary.py b/scripts/canary.py new file mode 100755 index 0000000..47016f7 --- /dev/null +++ b/scripts/canary.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +"""Fleet canary (G7.6). + +**Probes what a user does, not what is cheap to answer.** + +That distinction is the entire lesson of the 2026-08-12 outage: `/health` +returned 200 the whole time login was dead. A canary watching `/health` would +have stayed green for an hour while nobody in the ecosystem could sign in. So +every check here names a *user-visible* capability, and the login probe is the +one that matters most. + +Three rules this canary obeys: + +1. **Never report green for something it did not prove.** A check it could not + run reports `unknown`, never `ok` (I-8). +2. **Alert on transitions, not on every run.** A canary that emails every five + minutes gets filtered, and a filtered canary is a dead canary — which is how + the last one sat 37 days dead without anyone noticing. +3. **Run somewhere the thing being watched cannot take down with it.** This runs + on Veron 1 via Windy Git CI. A canary hosted on Kit 0 would die with Kit 0 + and report nothing at the exact moment it mattered. + +State lives in a small JSON file so consecutive runs can tell "still broken" +from "just broke". +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field + +STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json") +RESEND_KEY = os.environ.get("RESEND_API_KEY", "") +ALERT_TO = os.environ.get("CANARY_ALERT_TO", "grantwhitmer3@gmail.com") +ALERT_FROM = os.environ.get("CANARY_ALERT_FROM", "office@thewindstorm.uk") + +# Login is slow because account-server forks a node process per query. 18-25s is +# today's reality, not health. The threshold flags a real regression without +# crying wolf about the known-slow baseline; lower it as the adapter is fixed. +LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35")) +TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60")) + + +@dataclass +class Result: + name: str + status: str # ok | down | slow | unknown + detail: str + seconds: float = 0.0 + user_visible: str = "" + + +@dataclass +class Check: + name: str + url: str + what_it_proves: str + method: str = "GET" + body: dict | None = None + headers: dict = field(default_factory=dict) + warn_seconds: float | None = None + + +def _probe(c: Check) -> Result: + data = json.dumps(c.body).encode() if c.body else None + headers = {"User-Agent": "windy-git-canary/1.0", **c.headers} + if data: + headers["Content-Type"] = "application/json" + req = urllib.request.Request(c.url, data=data, method=c.method, headers=headers) + start = time.monotonic() + try: + with urllib.request.urlopen(req, timeout=TIMEOUT) as r: + elapsed = time.monotonic() - start + if r.status >= 400: + return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves) + warn = c.warn_seconds + if warn and elapsed > warn: + return Result( + c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)", + elapsed, c.what_it_proves, + ) + return Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves) + except urllib.error.HTTPError as e: + return Result(c.name, "down", f"HTTP {e.code}", time.monotonic() - start, c.what_it_proves) + except Exception as e: # noqa: BLE001 — a probe must never raise upward + return Result( + c.name, "down", f"{type(e).__name__}: {str(e)[:80]}", + time.monotonic() - start, c.what_it_proves, + ) + + +def build_checks() -> list[Check]: + checks = [ + Check( + "identity.health", + "https://account.windyword.ai/health", + "the identity service answers at all", + ), + Check( + "identity.jwks", + "https://account.windyword.ai/.well-known/jwks.json", + "every service can verify the tokens it is handed", + ), + Check( + "eternitas.health", + "https://api.eternitas.ai/health", + "agent passports can be issued and checked", + ), + Check( + "windygit.forge", + "https://app.windygit.com/api/v1/version", + "repositories are reachable", + ), + Check( + "windygit.plane", + "https://api.windygit.com/version", + "the Windy Git API answers", + ), + Check( + "dashboard", + "https://app.windyword.ai/", + "the dashboard loads", + ), + ] + + # THE important one. /health was 200 for the entire 2026-08-12 outage while + # this was timing out. A canary that skips it is decorative. + pw = os.environ.get("CANARY_LOGIN_PASSWORD", "") + email = os.environ.get("CANARY_LOGIN_EMAIL", "") + if pw and email: + checks.append( + Check( + "identity.login", + "https://account.windyword.ai/api/v1/auth/login", + "a human can actually sign in", + method="POST", + body={"email": email, "password": pw}, + warn_seconds=LOGIN_WARN_SECONDS, + ) + ) + return checks + + +def load_state() -> dict: + try: + with open(STATE_PATH) as f: + return json.load(f) + except (FileNotFoundError, json.JSONDecodeError): + return {} + + +def save_state(results: list[Result]) -> None: + with open(STATE_PATH, "w") as f: + json.dump({r.name: r.status for r in results}, f, indent=2) + + +def send_alert(subject: str, lines: list[str]) -> bool: + if not RESEND_KEY: + print("!! RESEND_API_KEY unset — cannot alert. This canary is decorative.") + return False + body = "\n".join(lines) + req = urllib.request.Request( + "https://api.resend.com/emails", + data=json.dumps({ + "from": f"Windy Canary <{ALERT_FROM}>", + "to": [ALERT_TO], + "subject": subject, + "text": body, + }).encode(), + method="POST", + headers={ + "Authorization": f"Bearer {RESEND_KEY}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=30) as r: + print(f" alert sent ({r.status})") + return True + except Exception as e: # noqa: BLE001 + print(f"!! alert FAILED: {e}") + return False + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--no-alert", action="store_true") + args = ap.parse_args() + + previous = load_state() + results = [_probe(c) for c in build_checks()] + + print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n") + for r in results: + mark = {"ok": " ok ", "slow": " SLOW ", "down": " DOWN ", "unknown": " ?? "}[r.status] + print(f"[{mark}] {r.name:20} {r.detail}") + if r.status != "ok": + print(f" ^ this means: {r.user_visible}") + + # Transitions only. "Still broken" does not re-alert; recovery does. + newly_bad = [r for r in results if r.status in ("down", "slow") and previous.get(r.name) == "ok"] + recovered = [ + r for r in results + if r.status == "ok" and previous.get(r.name) in ("down", "slow") + ] + + save_state(results) + + if not args.no_alert: + if newly_bad: + worst = "DOWN" if any(r.status == "down" for r in newly_bad) else "SLOW" + send_alert( + f"[Windy] {worst}: {', '.join(r.name for r in newly_bad)}", + [f"{r.name}: {r.detail}" for r in newly_bad] + + ["", "What this means for a person:"] + + [f" - {r.user_visible}" for r in newly_bad] + + ["", "Checked from Veron 1 via Windy Git CI — deliberately not from Kit 0."], + ) + if recovered: + send_alert( + f"[Windy] recovered: {', '.join(r.name for r in recovered)}", + [f"{r.name}: {r.detail}" for r in recovered], + ) + + # A failure exit makes the CI run red, so the forge itself carries the signal + # even if email is misconfigured. Two independent ways to notice. + return 1 if any(r.status in ("down", "slow") for r in results) else 0 + + +if __name__ == "__main__": + sys.exit(main())