ci: run dind under Sysbox, not privileged (rollback override kept)

dind was privileged: true, so a job that escaped into dind was root on
Veron 1, which is Grant's workstation. Under sysbox-runc (sysbox-ce 0.7.1,
installed 09-23 with no docker restart) dind root is an unprivileged host
uid. Smoke-tested standalone: nested containers, internet, a services-style
postgres on a private network and a python image all pass unprivileged.
Fresh volume dind-storage-sysbox; the old dind-storage stays for
docker-compose.privileged.yml, the one-command rollback.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

api/tests/test_canary_cleanup.py

@@ -1,92 +0,0 @@
-"""The canary's login probe must end the session it opens (journey cleanup rule)."""
-
-from __future__ import annotations
-
-import importlib.util
-import io
-import sys
-import urllib.error
-from pathlib import Path
-
-ROOT = Path(__file__).resolve().parents[2]
-_spec = importlib.util.spec_from_file_location("canary", ROOT / "scripts" / "canary.py")
-canary = importlib.util.module_from_spec(_spec)
-sys.modules["canary"] = canary  # dataclasses resolve their module by name
-_spec.loader.exec_module(canary)
-
-
-class _Resp:
-    def __init__(self, status=200, body=b"{}"):
-        self.status, self._body = status, body
-
-    def read(self):
-        return self._body
-
-    def __enter__(self):
-        return self
-
-    def __exit__(self, *a):
-        return False
-
-
-def _err(code):
-    return urllib.error.HTTPError(canary.LOGOUT_URL, code, "x", {}, io.BytesIO(b""))
-
-
-def _script(monkeypatch, outcomes):
-    calls = []
-
-    def fake(req, timeout=None):
-        calls.append((req.get_method(), req.full_url, req.get_header("Authorization")))
-        o = outcomes.pop(0)
-        if isinstance(o, Exception):
-            raise o
-        return o
-
-    monkeypatch.setattr(canary.urllib.request, "urlopen", fake)
-    return calls
-
-
-def test_logout_ends_the_session(monkeypatch):
-    calls = _script(monkeypatch, [_Resp(200)])
-    r = canary.logout("tok", sleep=lambda s: None)
-    assert r.status == "ok"
-    assert calls == [("POST", canary.LOGOUT_URL, "Bearer tok")]
-
-
-def test_5xx_and_no_response_are_retried_then_succeed(monkeypatch):
-    calls = _script(monkeypatch, [_err(502), OSError("reset"), _Resp(200)])
-    assert canary.logout("tok", sleep=lambda s: None).status == "ok"
-    assert len(calls) == 3
-
-
-def test_already_over_counts_as_done(monkeypatch):
-    _script(monkeypatch, [_err(401)])
-    assert canary.logout("tok", sleep=lambda s: None).status == "ok"
-
-
-def test_other_4xx_fails_fast_and_honestly(monkeypatch):
-    calls = _script(monkeypatch, [_err(400)])
-    r = canary.logout("tok", sleep=lambda s: None)
-    assert r.status == "down" and r.detail.startswith("CLEANUP FAILED") and len(calls) == 1
-
-
-def test_retries_are_bounded_and_reported(monkeypatch):
-    calls = _script(monkeypatch, [_err(503)] * 8)
-    r = canary.logout("tok", attempts=8, sleep=lambda s: None)
-    assert r.status == "down" and "CLEANUP FAILED after 8 tries" in r.detail and len(calls) == 8
-
-
-def test_login_probe_logs_out_with_the_token_it_got(monkeypatch):
-    calls = _script(monkeypatch, [_Resp(200, b'{"token": "abc"}'), _Resp(200)])
-    c = canary.Check("identity.login", "https://account.windyword.ai/api/v1/auth/login", "x",
-                     method="POST", body={"email": "e", "password": "p"},
-                     after=canary._logout_after_login)
-    r = canary._probe(c)
-    assert r.status == "ok" and [f.status for f in r.followups] == ["ok"]
-    assert calls[1] == ("POST", canary.LOGOUT_URL, "Bearer abc")
-
-
-def test_login_without_token_is_a_cleanup_failure_not_a_pass():
-    [f] = canary._logout_after_login(b"{}")
-    assert f.status == "down" and "CLEANUP FAILED" in f.detail

deploy/runner/docker-compose.privileged.yml

@@ -0,0 +1,15 @@
+# ROLLBACK ONLY: the pre-Sysbox dind (privileged: true), kept one command away.
+# Use it if CI breaks under Sysbox:
+#
+#   cd /srv/windygit/src/deploy/runner
+#   sudo docker compose -f docker-compose.yml -f docker-compose.privileged.yml up -d dind
+#
+# (then restart the runners while idle). Compose merges `volumes` by container
+# path, so this puts back the old `dind-storage` volume with its image cache.
+# Going forward again: the same command without the second -f.
+services:
+  dind:
+    runtime: runc
+    privileged: true
+    volumes:
+      - dind-storage:/var/lib/docker

deploy/runner/docker-compose.yml

@@ -26,8 +26,12 @@
 #   * `dind` and every job container it spawns are UNTRUSTED. They are on a
 #     private network with no access to the forge, its database, or its .env.
 #
-# dind itself is privileged — that is the cost, and it is the reason a job
-# escape lands in a disposable daemon rather than on Grant's workstation.
+# dind is NOT privileged (2026-09-23): it runs under the Sysbox runtime
+# (sysbox-ce on Veron, `runtime: sysbox-runc`), a user-namespaced system
+# container whose root is an unprivileged host uid. A job that escapes its own
+# container lands in dind as a nobody on the host, not as root on Grant's
+# workstation. Before Sysbox, dind was `privileged: true`; that config is kept
+# as docker-compose.privileged.yml (ROLLBACK ONLY, one command, see that file).
 #
 # ⚠️ Do NOT "simplify" this by mounting the host docker socket.
 
@@ -36,13 +40,15 @@ name: windy-git-runner
 services:
   dind:
     image: docker.io/library/docker:27-dind
-    privileged: true
+    runtime: sysbox-runc   # NOT privileged: see the I-5 note above
     environment:
       DOCKER_TLS_CERTDIR: ""   # plain TCP on an isolated network, no host route
     command: ["dockerd", "--host=tcp://0.0.0.0:2375", "--tls=false"]
     networks: [jobs]
     volumes:
-      - dind-storage:/var/lib/docker
+      # A fresh volume: Sysbox shifts ownership to its own uid range. The old
+      # `dind-storage` is kept untouched for the privileged rollback.
+      - dind-storage-sysbox:/var/lib/docker
     # G1.5 — bounded so a fork-bomb workflow cannot starve Grant's interactive
     # session. Veron 1 is his workstation, not a dedicated build box.
     cpus: 12.0            # 12 of 24 cores
@@ -159,6 +165,7 @@ networks:
 
 volumes:
   dind-storage:
+  dind-storage-sysbox:
   runner-data:
   runner-data-2:
   runner-data-3:

scripts/canary.py

@@ -33,7 +33,6 @@ import sys
 import time
 import urllib.error
 import urllib.request
-from collections.abc import Callable
 from dataclasses import dataclass, field
 
 STATE_PATH = os.environ.get("CANARY_STATE", "canary-state.json")
@@ -47,16 +46,6 @@ ALERT_FROM = os.environ.get("CANARY_ALERT_FROM", "office@thewindstorm.uk")
 LOGIN_WARN_SECONDS = float(os.environ.get("CANARY_LOGIN_WARN_S", "35"))
 TIMEOUT = float(os.environ.get("CANARY_TIMEOUT_S", "60"))
 
-# Journey cleanup rule (orchestrator, 2026-09-23). The login probe creates a hub
-# session (access + refresh token) every run, so it must end it. The hub's
-# /auth/logout revokes the token AND every refresh token of the account
-# (verified live: access 401, refresh 401 after it). So the next successful
-# logout also heals anything a failed run left behind; no ledger needed.
-LOGOUT_URL = "https://account.windyword.ai/api/v1/auth/logout"
-LOGOUT_ATTEMPTS = 8       # retried on 5xx / no response only
-LOGOUT_GAP_S = 15.0
-LOGOUT_GONE = (401, 404, 410)  # the session is already over = done
-
 
 @dataclass
 class Result:
@@ -65,7 +54,6 @@ class Result:
     detail: str
     seconds: float = 0.0
     user_visible: str = ""
-    followups: list[Result] = field(default_factory=list)
 
 
 @dataclass
@@ -80,8 +68,6 @@ class Check:
     # When True this check INVERTS: a 2xx is a critical failure (a security
     # control opened) and a 401/403/503 is the healthy, expected outcome.
     must_refuse: bool = False
-    # Runs on a 2xx with the response body; returns follow-up results (cleanup).
-    after: Callable[[bytes], list[Result]] | None = None
 
 
 def _probe(c: Check) -> Result:
@@ -105,18 +91,13 @@ def _probe(c: Check) -> Result:
                               elapsed, c.what_it_proves)
             if r.status >= 400:
                 return Result(c.name, "down", f"HTTP {r.status}", elapsed, c.what_it_proves)
-            raw = r.read()
             warn = c.warn_seconds
             if warn and elapsed > warn:
-                res = Result(
+                return Result(
                     c.name, "slow", f"HTTP {r.status} in {elapsed:.1f}s (warn >{warn:.0f}s)",
                     elapsed, c.what_it_proves,
                 )
-            else:
-                res = Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
-            if c.after:
-                res.followups = c.after(raw)
-            return res
+            return Result(c.name, "ok", f"HTTP {r.status} in {elapsed:.1f}s", elapsed, c.what_it_proves)
     except urllib.error.HTTPError as e:
         if c.must_refuse and e.code in (401, 403, 503):
             return Result(c.name, "ok", f"correctly refused (HTTP {e.code})",
@@ -129,52 +110,6 @@ def _probe(c: Check) -> Result:
         )
 
 
-def logout(token: str, *, attempts: int = LOGOUT_ATTEMPTS, gap: float = LOGOUT_GAP_S,
-           sleep: Callable[[float], None] = time.sleep) -> Result:
-    """End the session the login probe opened. Honest: never ok unless proven."""
-    what = "the canary leaves no live session behind (journey cleanup rule)"
-    headers = {
-        "User-Agent": "windy-git-canary/1.0",
-        "X-Windy-Synthetic": "1",
-        "Authorization": f"Bearer {token}",
-    }
-    start = time.monotonic()
-    last = "no attempt"
-    for i in range(attempts):
-        if i:
-            sleep(gap)
-        req = urllib.request.Request(LOGOUT_URL, data=b"", method="POST", headers=headers)
-        try:
-            with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
-                return Result("identity.logout", "ok", f"session ended (HTTP {r.status})",
-                              time.monotonic() - start, what)
-        except urllib.error.HTTPError as e:
-            if e.code in LOGOUT_GONE:
-                return Result("identity.logout", "ok", f"session already over (HTTP {e.code})",
-                              time.monotonic() - start, what)
-            if e.code < 500:  # a 4xx won't change on retry: fail fast
-                return Result("identity.logout", "down", f"CLEANUP FAILED: HTTP {e.code}",
-                              time.monotonic() - start, what)
-            last = f"HTTP {e.code}"
-        except Exception as e:  # noqa: BLE001 — no response / timeout: retry
-            last = f"{type(e).__name__}"
-    return Result("identity.logout", "down",
-                  f"CLEANUP FAILED after {attempts} tries: {last} (next run's logout heals it)",
-                  time.monotonic() - start, what)
-
-
-def _logout_after_login(raw: bytes) -> list[Result]:
-    try:
-        token = (json.loads(raw or b"{}") or {}).get("token")
-    except ValueError:
-        token = None
-    if not token:
-        return [Result("identity.logout", "down",
-                       "CLEANUP FAILED: login returned no token to log out with",
-                       0.0, "the canary leaves no live session behind (journey cleanup rule)")]
-    return [logout(token)]
-
-
 def build_checks() -> list[Check]:
     checks = [
         Check(
@@ -252,7 +187,6 @@ def build_checks() -> list[Check]:
                 method="POST",
                 body={"email": email, "password": pw},
                 warn_seconds=LOGIN_WARN_SECONDS,
-                after=_logout_after_login,
             )
         )
     return checks
@@ -329,10 +263,7 @@ def main() -> int:
     args = ap.parse_args()
 
     previous = load_state()
-    results = []
-    for c in build_checks():
-        r = _probe(c)
-        results += [r, *r.followups]
+    results = [_probe(c) for c in build_checks()]
 
     print(f"windy canary — {time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime())}\n")
     for r in results:

scripts/sync_from_github.sh

@@ -82,11 +82,7 @@ done
 
 # Jobs that name labels no runner has (ubuntu/macos/windows-latest) would wait
 # forever and invisibly; cancel them after 30 min. Never fails the sync.
-# Both DB steps go through `docker exec`, which hangs outright while the host
-# is in an IO stall (09-23: data2 SMR cliff wedged this sync for 10+ min and
-# stopped mirroring + the bridge for every lane). They are optional; mirroring
-# and the bridge are not. Bound them so a stuck exec costs one step, not the run.
-timeout -k 10 120 bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed or timed out (non-fatal)"
+bash "$(dirname "$0")/cancel_unrunnable.sh" || log "janitor failed (non-fatal)"
 
 # Private repos can't run GitHub Actions; mirror their open PRs here so CI
 # fires, and post the verdicts back to GitHub as commit statuses.
@@ -96,7 +92,7 @@ fi
 
 # CI telemetry -> admin.windyword.ai (shapes declared with Windy Telemetry 40).
 # Sends nothing until WINDYGIT_TELEMETRY_TOKEN is set; never fails the sync.
-timeout -k 10 180 python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed or timed out (non-fatal)"
+python3 "$(dirname "$0")/telemetry_emit.py" || log "telemetry emit failed (non-fatal)"
 
 [[ "$FAILED" -ne 0 ]] && { log "COMPLETED WITH FAILURES"; exit 1; }
 log "all repos in step with GitHub"