canary: end the hub session the login probe opens (journey cleanup rule)
identity.login created a live hub session every 10 min and never ended it. It now logs out with the token it got: retried on 5xx / no response (8 x 15 s), 401/404/410 = already over, any other 4xx fails fast, and a cleanup it can't finish is reported as identity.logout DOWN "CLEANUP FAILED" (alerts + red run). The hub's /auth/logout revokes every refresh token of the account (verified live), so the next run's logout heals a leftover; no ledger needed. Proven end to end: login 200, logout 200, 10/10 checks. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
92
api/tests/test_canary_cleanup.py
Normal file
92
api/tests/test_canary_cleanup.py
Normal file
@@ -0,0 +1,92 @@
|
|||||||
|
"""The canary's login probe must end the session it opens (journey cleanup rule)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import sys
|
||||||
|
import urllib.error
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
_spec = importlib.util.spec_from_file_location("canary", ROOT / "scripts" / "canary.py")
|
||||||
|
canary = importlib.util.module_from_spec(_spec)
|
||||||
|
sys.modules["canary"] = canary # dataclasses resolve their module by name
|
||||||
|
_spec.loader.exec_module(canary)
|
||||||
|
|
||||||
|
|
||||||
|
class _Resp:
|
||||||
|
def __init__(self, status=200, body=b"{}"):
|
||||||
|
self.status, self._body = status, body
|
||||||
|
|
||||||
|
def read(self):
|
||||||
|
return self._body
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *a):
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _err(code):
|
||||||
|
return urllib.error.HTTPError(canary.LOGOUT_URL, code, "x", {}, io.BytesIO(b""))
|
||||||
|
|
||||||
|
|
||||||
|
def _script(monkeypatch, outcomes):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def fake(req, timeout=None):
|
||||||
|
calls.append((req.get_method(), req.full_url, req.get_header("Authorization")))
|
||||||
|
o = outcomes.pop(0)
|
||||||
|
if isinstance(o, Exception):
|
||||||
|
raise o
|
||||||
|
return o
|
||||||
|
|
||||||
|
monkeypatch.setattr(canary.urllib.request, "urlopen", fake)
|
||||||
|
return calls
|
||||||
|
|
||||||
|
|
||||||
|
def test_logout_ends_the_session(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_Resp(200)])
|
||||||
|
r = canary.logout("tok", sleep=lambda s: None)
|
||||||
|
assert r.status == "ok"
|
||||||
|
assert calls == [("POST", canary.LOGOUT_URL, "Bearer tok")]
|
||||||
|
|
||||||
|
|
||||||
|
def test_5xx_and_no_response_are_retried_then_succeed(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(502), OSError("reset"), _Resp(200)])
|
||||||
|
assert canary.logout("tok", sleep=lambda s: None).status == "ok"
|
||||||
|
assert len(calls) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_already_over_counts_as_done(monkeypatch):
|
||||||
|
_script(monkeypatch, [_err(401)])
|
||||||
|
assert canary.logout("tok", sleep=lambda s: None).status == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
def test_other_4xx_fails_fast_and_honestly(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(400)])
|
||||||
|
r = canary.logout("tok", sleep=lambda s: None)
|
||||||
|
assert r.status == "down" and r.detail.startswith("CLEANUP FAILED") and len(calls) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_retries_are_bounded_and_reported(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_err(503)] * 8)
|
||||||
|
r = canary.logout("tok", attempts=8, sleep=lambda s: None)
|
||||||
|
assert r.status == "down" and "CLEANUP FAILED after 8 tries" in r.detail and len(calls) == 8
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_probe_logs_out_with_the_token_it_got(monkeypatch):
|
||||||
|
calls = _script(monkeypatch, [_Resp(200, b'{"token": "abc"}'), _Resp(200)])
|
||||||
|
c = canary.Check("identity.login", "https://account.windyword.ai/api/v1/auth/login", "x",
|
||||||
|
method="POST", body={"email": "e", "password": "p"},
|
||||||
|
after=canary._logout_after_login)
|
||||||
|
r = canary._probe(c)
|
||||||
|
assert r.status == "ok" and [f.status for f in r.followups] == ["ok"]
|
||||||
|
assert calls[1] == ("POST", canary.LOGOUT_URL, "Bearer abc")
|
||||||
|
|
||||||
|
|
||||||
|
def test_login_without_token_is_a_cleanup_failure_not_a_pass():
|
||||||
|
[f] = canary._logout_after_login(b"{}")
|
||||||
|
assert f.status == "down" and "CLEANUP FAILED" in f.detail
|
||||||
@@ -33,6 +33,7 @@ import sys
|
|||||||
import time
|
import time
|
||||||
import urllib.error
|
import urllib.error
|
||||||
import urllib.request
|
import urllib.request
|
||||||
|
from collections.abc import Callable
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
|
|
||||||
STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json")
|
STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json")
|
||||||
@@ -46,6 +47,16 @@ ALERT_FROM = os.environ.get("CANARY_ALERT_FROM", "office@thewindstorm.uk")
|
|||||||
LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35"))
|
LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35"))
|
||||||
TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60"))
|
TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60"))
|
||||||
|
|
||||||
|
# Journey cleanup rule (orchestrator, 2026-09-23). The login probe creates a hub
|
||||||
|
# session (access + refresh token) every run, so it must end it. The hub's
|
||||||
|
# /auth/logout revokes the token AND every refresh token of the account
|
||||||
|
# (verified live: access 401, refresh 401 after it). So the next successful
|
||||||
|
# logout also heals anything a failed run left behind; no ledger needed.
|
||||||
|
LOGOUT_URL = "https://account.windyword.ai/api/v1/auth/logout"
|
||||||
|
LOGOUT_ATTEMPTS = 8 # retried on 5xx / no response only
|
||||||
|
LOGOUT_GAP_S = 15.0
|
||||||
|
LOGOUT_GONE = (401, 404, 410) # the session is already over = done
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class Result:
|
class Result:
|
||||||
@@ -54,6 +65,7 @@ class Result:
|
|||||||
detail: str
|
detail: str
|
||||||
seconds: float = 0.0
|
seconds: float = 0.0
|
||||||
user_visible: str = ""
|
user_visible: str = ""
|
||||||
|
followups: list[Result] = field(default_factory=list)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -68,6 +80,8 @@ class Check:
|
|||||||
# When True this check INVERTS: a 2xx is a critical failure (a security
|
# When True this check INVERTS: a 2xx is a critical failure (a security
|
||||||
# control opened) and a 401/403/503 is the healthy, expected outcome.
|
# control opened) and a 401/403/503 is the healthy, expected outcome.
|
||||||
must_refuse: bool = False
|
must_refuse: bool = False
|
||||||
|
# Runs on a 2xx with the response body; returns follow-up results (cleanup).
|
||||||
|
after: Callable[[bytes], list[Result]] | None = None
|
||||||
|
|
||||||
|
|
||||||
def _probe(c: Check) -> Result:
|
def _probe(c: Check) -> Result:
|
||||||
@@ -91,13 +105,18 @@ def _probe(c: Check) -> Result:
|
|||||||
elapsed, c.what_it_proves)
|
elapsed, c.what_it_proves)
|
||||||
if r.status >= 400:
|
if r.status >= 400:
|
||||||
return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves)
|
return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves)
|
||||||
|
raw = r.read()
|
||||||
warn = c.warn_seconds
|
warn = c.warn_seconds
|
||||||
if warn and elapsed > warn:
|
if warn and elapsed > warn:
|
||||||
return Result(
|
res = Result(
|
||||||
c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)",
|
c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)",
|
||||||
elapsed, c.what_it_proves,
|
elapsed, c.what_it_proves,
|
||||||
)
|
)
|
||||||
return Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
|
else:
|
||||||
|
res = Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
|
||||||
|
if c.after:
|
||||||
|
res.followups = c.after(raw)
|
||||||
|
return res
|
||||||
except urllib.error.HTTPError as e:
|
except urllib.error.HTTPError as e:
|
||||||
if c.must_refuse and e.code in (401, 403, 503):
|
if c.must_refuse and e.code in (401, 403, 503):
|
||||||
return Result(c.name, "ok", f"correctly refused (HTTP {e.code})",
|
return Result(c.name, "ok", f"correctly refused (HTTP {e.code})",
|
||||||
@@ -110,6 +129,52 @@ def _probe(c: Check) -> Result:
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def logout(token: str, *, attempts: int = LOGOUT_ATTEMPTS, gap: float = LOGOUT_GAP_S,
|
||||||
|
sleep: Callable[[float], None] = time.sleep) -> Result:
|
||||||
|
"""End the session the login probe opened. Honest: never ok unless proven."""
|
||||||
|
what = "the canary leaves no live session behind (journey cleanup rule)"
|
||||||
|
headers = {
|
||||||
|
"User-Agent": "windy-git-canary/1.0",
|
||||||
|
"X-Windy-Synthetic": "1",
|
||||||
|
"Authorization": f"Bearer {token}",
|
||||||
|
}
|
||||||
|
start = time.monotonic()
|
||||||
|
last = "no attempt"
|
||||||
|
for i in range(attempts):
|
||||||
|
if i:
|
||||||
|
sleep(gap)
|
||||||
|
req = urllib.request.Request(LOGOUT_URL, data=b"", method="POST", headers=headers)
|
||||||
|
try:
|
||||||
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
|
||||||
|
return Result("identity.logout", "ok", f"session ended (HTTP {r.status})",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
except urllib.error.HTTPError as e:
|
||||||
|
if e.code in LOGOUT_GONE:
|
||||||
|
return Result("identity.logout", "ok", f"session already over (HTTP {e.code})",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
if e.code < 500: # a 4xx won't change on retry: fail fast
|
||||||
|
return Result("identity.logout", "down", f"CLEANUP FAILED: HTTP {e.code}",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
last = f"HTTP {e.code}"
|
||||||
|
except Exception as e: # noqa: BLE001 — no response / timeout: retry
|
||||||
|
last = f"{type(e).__name__}"
|
||||||
|
return Result("identity.logout", "down",
|
||||||
|
f"CLEANUP FAILED after {attempts} tries: {last} (next run's logout heals it)",
|
||||||
|
time.monotonic() - start, what)
|
||||||
|
|
||||||
|
|
||||||
|
def _logout_after_login(raw: bytes) -> list[Result]:
|
||||||
|
try:
|
||||||
|
token = (json.loads(raw or b"{}") or {}).get("token")
|
||||||
|
except ValueError:
|
||||||
|
token = None
|
||||||
|
if not token:
|
||||||
|
return [Result("identity.logout", "down",
|
||||||
|
"CLEANUP FAILED: login returned no token to log out with",
|
||||||
|
0.0, "the canary leaves no live session behind (journey cleanup rule)")]
|
||||||
|
return [logout(token)]
|
||||||
|
|
||||||
|
|
||||||
def build_checks() -> list[Check]:
|
def build_checks() -> list[Check]:
|
||||||
checks = [
|
checks = [
|
||||||
Check(
|
Check(
|
||||||
@@ -187,6 +252,7 @@ def build_checks() -> list[Check]:
|
|||||||
method="POST",
|
method="POST",
|
||||||
body={"email": email, "password": pw},
|
body={"email": email, "password": pw},
|
||||||
warn_seconds=LOGIN_WARN_SECONDS,
|
warn_seconds=LOGIN_WARN_SECONDS,
|
||||||
|
after=_logout_after_login,
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
return checks
|
return checks
|
||||||
@@ -263,7 +329,10 @@ def main() -> int:
|
|||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
previous = load_state()
|
previous = load_state()
|
||||||
results = [_probe(c) for c in build_checks()]
|
results = []
|
||||||
|
for c in build_checks():
|
||||||
|
r = _probe(c)
|
||||||
|
results += [r, *r.followups]
|
||||||
|
|
||||||
print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n")
|
print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n")
|
||||||
for r in results:
|
for r in results:
|
||||||
|
|||||||
Reference in New Issue
Block a user